git.squeep.com Git - akkoma/blob - benchmarks/load_testing/generator.ex

   1 defmodule Pleroma.LoadTesting.Generator do
   2   use Pleroma.LoadTesting.Helper
   3   alias Pleroma.Web.CommonAPI
   4
   5   def generate_like_activities(user, posts) do
   6     count_likes = Kernel.trunc(length(posts) / 4)
   7     IO.puts("Starting generating #{count_likes} like activities...")
   8
   9     {time, _} =
  10       :timer.tc(fn ->
  11         Task.async_stream(
  12           Enum.take_random(posts, count_likes),
  13           fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
  14           max_concurrency: 10,
  15           timeout: 30_000
  16         )
  17         |> Stream.run()
  18       end)
  19
  20     IO.puts("Inserting like activities take #{to_sec(time)} sec.\n")
  21   end
  22
  23   def generate_users(opts) do
  24     IO.puts("Starting generating #{opts[:users_max]} users...")
  25     {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
  26
  27     IO.puts("Inserting users take #{to_sec(time)} sec.\n")
  28   end
  29
  30   defp do_generate_users(opts) do
  31     max = Keyword.get(opts, :users_max)
  32
  33     Task.async_stream(
  34       1..max,
  35       &generate_user_data(&1),
  36       max_concurrency: 10,
  37       timeout: 30_000
  38     )
  39     |> Enum.to_list()
  40   end
  41
  42   defp generate_user_data(i) do
  43     remote = Enum.random([true, false])
  44
  45     user = %User{
  46       name: "Test テスト User #{i}",
  47       email: "user#{i}@example.com",
  48       nickname: "nick#{i}",
  49       password_hash:
  50         "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
  51       bio: "Tester Number #{i}",
  52       local: remote
  53     }
  54
  55     user_urls =
  56       if remote do
  57         base_url =
  58           Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
  59
  60         ap_id = "#{base_url}/users/#{user.nickname}"
  61
  62         %{
  63           ap_id: ap_id,
  64           follower_address: ap_id <> "/followers",
  65           following_address: ap_id <> "/following"
  66         }
  67       else
  68         %{
  69           ap_id: User.ap_id(user),
  70           follower_address: User.ap_followers(user),
  71           following_address: User.ap_following(user)
  72         }
  73       end
  74
  75     user = Map.merge(user, user_urls)
  76
  77     Repo.insert!(user)
  78   end
  79
  80   def generate_activities(user, users) do
  81     do_generate_activities(user, users)
  82   end
  83
  84   defp do_generate_activities(user, users) do
  85     IO.puts("Starting generating 20000 common activities...")
  86
  87     {time, _} =
  88       :timer.tc(fn ->
  89         Task.async_stream(
  90           1..20_000,
  91           fn _ ->
  92             do_generate_activity([user | users])
  93           end,
  94           max_concurrency: 10,
  95           timeout: 30_000
  96         )
  97         |> Stream.run()
  98       end)
  99
 100     IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
 101
 102     IO.puts("Starting generating 20000 activities with mentions...")
 103
 104     {time, _} =
 105       :timer.tc(fn ->
 106         Task.async_stream(
 107           1..20_000,
 108           fn _ ->
 109             do_generate_activity_with_mention(user, users)
 110           end,
 111           max_concurrency: 10,
 112           timeout: 30_000
 113         )
 114         |> Stream.run()
 115       end)
 116
 117     IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
 118
 119     IO.puts("Starting generating 10000 activities with threads...")
 120
 121     {time, _} =
 122       :timer.tc(fn ->
 123         Task.async_stream(
 124           1..10_000,
 125           fn _ ->
 126             do_generate_threads([user | users])
 127           end,
 128           max_concurrency: 10,
 129           timeout: 30_000
 130         )
 131         |> Stream.run()
 132       end)
 133
 134     IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
 135   end
 136
 137   defp do_generate_activity(users) do
 138     post = %{
 139       "status" => "Some status without mention with random user"
 140     }
 141
 142     CommonAPI.post(Enum.random(users), post)
 143   end
 144
 145   def generate_power_intervals(opts \\ []) do
 146     count = Keyword.get(opts, :count, 20)
 147     power = Keyword.get(opts, :power, 2)
 148     IO.puts("Generating #{count} intervals for a power #{power} series...")
 149     counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
 150     sum = Enum.sum(counts)
 151
 152     densities =
 153       Enum.map(counts, fn c ->
 154         c / sum
 155       end)
 156
 157     densities
 158     |> Enum.reduce(0, fn density, acc ->
 159       if acc == 0 do
 160         [{0, density}]
 161       else
 162         [{_, lower} | _] = acc
 163         [{lower, lower + density} | acc]
 164       end
 165     end)
 166     |> Enum.reverse()
 167   end
 168
 169   def generate_tagged_activities(opts \\ []) do
 170     tag_count = Keyword.get(opts, :tag_count, 20)
 171     users = Keyword.get(opts, :users, Repo.all(User))
 172     activity_count = Keyword.get(opts, :count, 200_000)
 173
 174     intervals = generate_power_intervals(count: tag_count)
 175
 176     IO.puts(
 177       "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
 178     )
 179
 180     Enum.each(1..activity_count, fn _ ->
 181       random = :rand.uniform()
 182       i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
 183       CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
 184     end)
 185   end
 186
 187   defp do_generate_activity_with_mention(user, users) do
 188     mentions_cnt = Enum.random([2, 3, 4, 5])
 189     with_user = Enum.random([true, false])
 190     users = Enum.shuffle(users)
 191     mentions_users = Enum.take(users, mentions_cnt)
 192     mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
 193
 194     mentions_str =
 195       Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
 196
 197     post = %{
 198       "status" => mentions_str <> "some status with mentions random users"
 199     }
 200
 201     CommonAPI.post(Enum.random(users), post)
 202   end
 203
 204   defp do_generate_threads(users) do
 205     thread_length = Enum.random([2, 3, 4, 5])
 206     actor = Enum.random(users)
 207
 208     post = %{
 209       "status" => "Start of the thread"
 210     }
 211
 212     {:ok, activity} = CommonAPI.post(actor, post)
 213
 214     Enum.each(1..thread_length, fn _ ->
 215       user = Enum.random(users)
 216
 217       post = %{
 218         "status" => "@#{actor.nickname} reply to thread",
 219         "in_reply_to_status_id" => activity.id
 220       }
 221
 222       CommonAPI.post(user, post)
 223     end)
 224   end
 225
 226   def generate_remote_activities(user, users) do
 227     do_generate_remote_activities(user, users)
 228   end
 229
 230   defp do_generate_remote_activities(user, users) do
 231     IO.puts("Starting generating 10000 remote activities...")
 232
 233     {time, _} =
 234       :timer.tc(fn ->
 235         Task.async_stream(
 236           1..10_000,
 237           fn i ->
 238             do_generate_remote_activity(i, user, users)
 239           end,
 240           max_concurrency: 10,
 241           timeout: 30_000
 242         )
 243         |> Stream.run()
 244       end)
 245
 246     IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
 247   end
 248
 249   defp do_generate_remote_activity(i, user, users) do
 250     actor = Enum.random(users)
 251     %{host: host} = URI.parse(actor.ap_id)
 252     date = Date.utc_today()
 253     datetime = DateTime.utc_now()
 254
 255     map = %{
 256       "actor" => actor.ap_id,
 257       "cc" => [actor.follower_address, user.ap_id],
 258       "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
 259       "id" => actor.ap_id <> "/statuses/#{i}/activity",
 260       "object" => %{
 261         "actor" => actor.ap_id,
 262         "atomUri" => actor.ap_id <> "/statuses/#{i}",
 263         "attachment" => [],
 264         "attributedTo" => actor.ap_id,
 265         "bcc" => [],
 266         "bto" => [],
 267         "cc" => [actor.follower_address, user.ap_id],
 268         "content" =>
 269           "<p><span class=\"h-card\"><a href=\"" <>
 270             user.ap_id <>
 271             "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
 272         "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
 273         "conversation" =>
 274           "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
 275         "emoji" => %{},
 276         "id" => actor.ap_id <> "/statuses/#{i}",
 277         "inReplyTo" => nil,
 278         "inReplyToAtomUri" => nil,
 279         "published" => datetime,
 280         "sensitive" => true,
 281         "summary" => "cw",
 282         "tag" => [
 283           %{
 284             "href" => user.ap_id,
 285             "name" => "@#{user.nickname}@#{host}",
 286             "type" => "Mention"
 287           }
 288         ],
 289         "to" => ["https://www.w3.org/ns/activitystreams#Public"],
 290         "type" => "Note",
 291         "url" => "http://#{host}/@#{actor.nickname}/#{i}"
 292       },
 293       "published" => datetime,
 294       "to" => ["https://www.w3.org/ns/activitystreams#Public"],
 295       "type" => "Create"
 296     }
 297
 298     Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
 299   end
 300
 301   def generate_dms(user, users, opts) do
 302     IO.puts("Starting generating #{opts[:dms_max]} DMs")
 303     {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
 304     IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
 305   end
 306
 307   defp do_generate_dms(user, users, opts) do
 308     Task.async_stream(
 309       1..opts[:dms_max],
 310       fn _ ->
 311         do_generate_dm(user, users)
 312       end,
 313       max_concurrency: 10,
 314       timeout: 30_000
 315     )
 316     |> Stream.run()
 317   end
 318
 319   defp do_generate_dm(user, users) do
 320     post = %{
 321       "status" => "@#{user.nickname} some direct message",
 322       "visibility" => "direct"
 323     }
 324
 325     CommonAPI.post(Enum.random(users), post)
 326   end
 327
 328   def generate_long_thread(user, users, opts) do
 329     IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
 330     {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
 331     IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
 332     {:ok, activity}
 333   end
 334
 335   defp do_generate_long_thread(user, users, opts) do
 336     {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
 337
 338     Task.async_stream(
 339       1..opts[:thread_length],
 340       fn _ -> do_generate_thread(users, id) end,
 341       max_concurrency: 10,
 342       timeout: 30_000
 343     )
 344     |> Stream.run()
 345
 346     activity
 347   end
 348
 349   defp do_generate_thread(users, activity_id) do
 350     CommonAPI.post(Enum.random(users), %{
 351       "status" => "reply to main post",
 352       "in_reply_to_status_id" => activity_id
 353     })
 354   end
 355
 356   def generate_non_visible_message(user, users) do
 357     IO.puts("Starting generating 1000 non visible posts")
 358
 359     {time, _} =
 360       :timer.tc(fn ->
 361         do_generate_non_visible_posts(user, users)
 362       end)
 363
 364     IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
 365   end
 366
 367   defp do_generate_non_visible_posts(user, users) do
 368     [not_friend | users] = users
 369
 370     make_friends(user, users)
 371
 372     Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
 373       max_concurrency: 10,
 374       timeout: 30_000
 375     )
 376     |> Stream.run()
 377   end
 378
 379   defp make_friends(_user, []), do: nil
 380
 381   defp make_friends(user, [friend | users]) do
 382     {:ok, _} = User.follow(user, friend)
 383     {:ok, _} = User.follow(friend, user)
 384     make_friends(user, users)
 385   end
 386
 387   defp do_generate_non_visible_post(not_friend, users) do
 388     post = %{
 389       "status" => "some non visible post",
 390       "visibility" => "private"
 391     }
 392
 393     {:ok, activity} = CommonAPI.post(not_friend, post)
 394
 395     thread_length = Enum.random([2, 3, 4, 5])
 396
 397     Enum.each(1..thread_length, fn _ ->
 398       user = Enum.random(users)
 399
 400       post = %{
 401         "status" => "@#{not_friend.nickname} reply to non visible post",
 402         "in_reply_to_status_id" => activity.id,
 403         "visibility" => "private"
 404       }
 405
 406       CommonAPI.post(user, post)
 407     end)
 408   end
 409 end