git.squeep.com Git - akkoma/blob - benchmarks/load_testing/generator.ex

   1 defmodule Pleroma.LoadTesting.Generator do
   2   use Pleroma.LoadTesting.Helper
   3   alias Pleroma.Web.CommonAPI
   4
   5   def generate_like_activities(user, posts) do
   6     count_likes = Kernel.trunc(length(posts) / 4)
   7     IO.puts("Starting generating #{count_likes} like activities...")
   8
   9     {time, _} =
  10       :timer.tc(fn ->
  11         Task.async_stream(
  12           Enum.take_random(posts, count_likes),
  13           fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
  14           max_concurrency: 10,
  15           timeout: 30_000
  16         )
  17         |> Stream.run()
  18       end)
  19
  20     IO.puts("Inserting like activities take #{to_sec(time)} sec.\n")
  21   end
  22
  23   def generate_users(opts) do
  24     IO.puts("Starting generating #{opts[:users_max]} users...")
  25     {time, users} = :timer.tc(fn -> do_generate_users(opts) end)
  26
  27     IO.puts("Inserting users took #{to_sec(time)} sec.\n")
  28     users
  29   end
  30
  31   defp do_generate_users(opts) do
  32     max = Keyword.get(opts, :users_max)
  33
  34     Task.async_stream(
  35       1..max,
  36       &generate_user_data(&1),
  37       max_concurrency: 10,
  38       timeout: 30_000
  39     )
  40     |> Enum.to_list()
  41   end
  42
  43   defp generate_user_data(i) do
  44     remote = Enum.random([true, false])
  45
  46     user = %User{
  47       name: "Test テスト User #{i}",
  48       email: "user#{i}@example.com",
  49       nickname: "nick#{i}",
  50       password_hash:
  51         "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
  52       bio: "Tester Number #{i}",
  53       local: remote
  54     }
  55
  56     user_urls =
  57       if remote do
  58         base_url =
  59           Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
  60
  61         ap_id = "#{base_url}/users/#{user.nickname}"
  62
  63         %{
  64           ap_id: ap_id,
  65           follower_address: ap_id <> "/followers",
  66           following_address: ap_id <> "/following"
  67         }
  68       else
  69         %{
  70           ap_id: User.ap_id(user),
  71           follower_address: User.ap_followers(user),
  72           following_address: User.ap_following(user)
  73         }
  74       end
  75
  76     user = Map.merge(user, user_urls)
  77
  78     Repo.insert!(user)
  79   end
  80
  81   def generate_activities(user, users) do
  82     do_generate_activities(user, users)
  83   end
  84
  85   defp do_generate_activities(user, users) do
  86     IO.puts("Starting generating 20000 common activities...")
  87
  88     {time, _} =
  89       :timer.tc(fn ->
  90         Task.async_stream(
  91           1..20_000,
  92           fn _ ->
  93             do_generate_activity([user | users])
  94           end,
  95           max_concurrency: 10,
  96           timeout: 30_000
  97         )
  98         |> Stream.run()
  99       end)
 100
 101     IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
 102
 103     IO.puts("Starting generating 20000 activities with mentions...")
 104
 105     {time, _} =
 106       :timer.tc(fn ->
 107         Task.async_stream(
 108           1..20_000,
 109           fn _ ->
 110             do_generate_activity_with_mention(user, users)
 111           end,
 112           max_concurrency: 10,
 113           timeout: 30_000
 114         )
 115         |> Stream.run()
 116       end)
 117
 118     IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
 119
 120     IO.puts("Starting generating 10000 activities with threads...")
 121
 122     {time, _} =
 123       :timer.tc(fn ->
 124         Task.async_stream(
 125           1..10_000,
 126           fn _ ->
 127             do_generate_threads([user | users])
 128           end,
 129           max_concurrency: 10,
 130           timeout: 30_000
 131         )
 132         |> Stream.run()
 133       end)
 134
 135     IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
 136   end
 137
 138   defp do_generate_activity(users) do
 139     post = %{
 140       "status" => "Some status without mention with random user"
 141     }
 142
 143     CommonAPI.post(Enum.random(users), post)
 144   end
 145
 146   def generate_power_intervals(opts \\ []) do
 147     count = Keyword.get(opts, :count, 20)
 148     power = Keyword.get(opts, :power, 2)
 149     IO.puts("Generating #{count} intervals for a power #{power} series...")
 150     counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
 151     sum = Enum.sum(counts)
 152
 153     densities =
 154       Enum.map(counts, fn c ->
 155         c / sum
 156       end)
 157
 158     densities
 159     |> Enum.reduce(0, fn density, acc ->
 160       if acc == 0 do
 161         [{0, density}]
 162       else
 163         [{_, lower} | _] = acc
 164         [{lower, lower + density} | acc]
 165       end
 166     end)
 167     |> Enum.reverse()
 168   end
 169
 170   def generate_tagged_activities(opts \\ []) do
 171     tag_count = Keyword.get(opts, :tag_count, 20)
 172     users = Keyword.get(opts, :users, Repo.all(User))
 173     activity_count = Keyword.get(opts, :count, 200_000)
 174
 175     intervals = generate_power_intervals(count: tag_count)
 176
 177     IO.puts(
 178       "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
 179     )
 180
 181     Enum.each(1..activity_count, fn _ ->
 182       random = :rand.uniform()
 183       i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
 184       CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
 185     end)
 186   end
 187
 188   defp do_generate_activity_with_mention(user, users) do
 189     mentions_cnt = Enum.random([2, 3, 4, 5])
 190     with_user = Enum.random([true, false])
 191     users = Enum.shuffle(users)
 192     mentions_users = Enum.take(users, mentions_cnt)
 193     mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
 194
 195     mentions_str =
 196       Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
 197
 198     post = %{
 199       "status" => mentions_str <> "some status with mentions random users"
 200     }
 201
 202     CommonAPI.post(Enum.random(users), post)
 203   end
 204
 205   defp do_generate_threads(users) do
 206     thread_length = Enum.random([2, 3, 4, 5])
 207     actor = Enum.random(users)
 208
 209     post = %{
 210       "status" => "Start of the thread"
 211     }
 212
 213     {:ok, activity} = CommonAPI.post(actor, post)
 214
 215     Enum.each(1..thread_length, fn _ ->
 216       user = Enum.random(users)
 217
 218       post = %{
 219         "status" => "@#{actor.nickname} reply to thread",
 220         "in_reply_to_status_id" => activity.id
 221       }
 222
 223       CommonAPI.post(user, post)
 224     end)
 225   end
 226
 227   def generate_remote_activities(user, users) do
 228     do_generate_remote_activities(user, users)
 229   end
 230
 231   defp do_generate_remote_activities(user, users) do
 232     IO.puts("Starting generating 10000 remote activities...")
 233
 234     {time, _} =
 235       :timer.tc(fn ->
 236         Task.async_stream(
 237           1..10_000,
 238           fn i ->
 239             do_generate_remote_activity(i, user, users)
 240           end,
 241           max_concurrency: 10,
 242           timeout: 30_000
 243         )
 244         |> Stream.run()
 245       end)
 246
 247     IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
 248   end
 249
 250   defp do_generate_remote_activity(i, user, users) do
 251     actor = Enum.random(users)
 252     %{host: host} = URI.parse(actor.ap_id)
 253     date = Date.utc_today()
 254     datetime = DateTime.utc_now()
 255
 256     map = %{
 257       "actor" => actor.ap_id,
 258       "cc" => [actor.follower_address, user.ap_id],
 259       "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
 260       "id" => actor.ap_id <> "/statuses/#{i}/activity",
 261       "object" => %{
 262         "actor" => actor.ap_id,
 263         "atomUri" => actor.ap_id <> "/statuses/#{i}",
 264         "attachment" => [],
 265         "attributedTo" => actor.ap_id,
 266         "bcc" => [],
 267         "bto" => [],
 268         "cc" => [actor.follower_address, user.ap_id],
 269         "content" =>
 270           "<p><span class=\"h-card\"><a href=\"" <>
 271             user.ap_id <>
 272             "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
 273         "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
 274         "conversation" =>
 275           "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
 276         "emoji" => %{},
 277         "id" => actor.ap_id <> "/statuses/#{i}",
 278         "inReplyTo" => nil,
 279         "inReplyToAtomUri" => nil,
 280         "published" => datetime,
 281         "sensitive" => true,
 282         "summary" => "cw",
 283         "tag" => [
 284           %{
 285             "href" => user.ap_id,
 286             "name" => "@#{user.nickname}@#{host}",
 287             "type" => "Mention"
 288           }
 289         ],
 290         "to" => ["https://www.w3.org/ns/activitystreams#Public"],
 291         "type" => "Note",
 292         "url" => "http://#{host}/@#{actor.nickname}/#{i}"
 293       },
 294       "published" => datetime,
 295       "to" => ["https://www.w3.org/ns/activitystreams#Public"],
 296       "type" => "Create"
 297     }
 298
 299     Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
 300   end
 301
 302   def generate_dms(user, users, opts) do
 303     IO.puts("Starting generating #{opts[:dms_max]} DMs")
 304     {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
 305     IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
 306   end
 307
 308   defp do_generate_dms(user, users, opts) do
 309     Task.async_stream(
 310       1..opts[:dms_max],
 311       fn _ ->
 312         do_generate_dm(user, users)
 313       end,
 314       max_concurrency: 10,
 315       timeout: 30_000
 316     )
 317     |> Stream.run()
 318   end
 319
 320   defp do_generate_dm(user, users) do
 321     post = %{
 322       "status" => "@#{user.nickname} some direct message",
 323       "visibility" => "direct"
 324     }
 325
 326     CommonAPI.post(Enum.random(users), post)
 327   end
 328
 329   def generate_long_thread(user, users, opts) do
 330     IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
 331     {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
 332     IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
 333     {:ok, activity}
 334   end
 335
 336   defp do_generate_long_thread(user, users, opts) do
 337     {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
 338
 339     Task.async_stream(
 340       1..opts[:thread_length],
 341       fn _ -> do_generate_thread(users, id) end,
 342       max_concurrency: 10,
 343       timeout: 30_000
 344     )
 345     |> Stream.run()
 346
 347     activity
 348   end
 349
 350   defp do_generate_thread(users, activity_id) do
 351     CommonAPI.post(Enum.random(users), %{
 352       "status" => "reply to main post",
 353       "in_reply_to_status_id" => activity_id
 354     })
 355   end
 356
 357   def generate_non_visible_message(user, users) do
 358     IO.puts("Starting generating 1000 non visible posts")
 359
 360     {time, _} =
 361       :timer.tc(fn ->
 362         do_generate_non_visible_posts(user, users)
 363       end)
 364
 365     IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
 366   end
 367
 368   defp do_generate_non_visible_posts(user, users) do
 369     [not_friend | users] = users
 370
 371     make_friends(user, users)
 372
 373     Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
 374       max_concurrency: 10,
 375       timeout: 30_000
 376     )
 377     |> Stream.run()
 378   end
 379
 380   defp make_friends(_user, []), do: nil
 381
 382   defp make_friends(user, [friend | users]) do
 383     {:ok, _} = User.follow(user, friend)
 384     {:ok, _} = User.follow(friend, user)
 385     make_friends(user, users)
 386   end
 387
 388   defp do_generate_non_visible_post(not_friend, users) do
 389     post = %{
 390       "status" => "some non visible post",
 391       "visibility" => "private"
 392     }
 393
 394     {:ok, activity} = CommonAPI.post(not_friend, post)
 395
 396     thread_length = Enum.random([2, 3, 4, 5])
 397
 398     Enum.each(1..thread_length, fn _ ->
 399       user = Enum.random(users)
 400
 401       post = %{
 402         "status" => "@#{not_friend.nickname} reply to non visible post",
 403         "in_reply_to_status_id" => activity.id,
 404         "visibility" => "private"
 405       }
 406
 407       CommonAPI.post(user, post)
 408     end)
 409   end
 410 end