1 # Pleroma: A lightweight social networking server
2 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
3 # SPDX-License-Identifier: AGPL-3.0-only
5 defmodule Mix.Tasks.Pleroma.DatabaseTest do
6 use Pleroma.DataCase, async: true
7 use Oban.Testing, repo: Pleroma.Repo
13 alias Pleroma.Web.CommonAPI
15 import Pleroma.Factory
18 Mix.shell(Mix.Shell.Process)
21 Mix.shell(Mix.Shell.IO)
27 describe "running remove_embedded_objects" do
28 test "it replaces objects with references" do
30 {:ok, activity} = CommonAPI.post(user, %{status: "test"})
31 new_data = Map.put(activity.data, "object", activity.object.data)
35 |> Activity.change(%{data: new_data})
38 assert is_map(activity.data["object"])
40 Mix.Tasks.Pleroma.Database.run(["remove_embedded_objects"])
42 activity = Activity.get_by_id_with_object(activity.id)
43 assert is_binary(activity.data["object"])
47 describe "prune_objects" do
48 test "it prunes old objects from the database" do
49 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
53 |> Timex.shift(days: -deadline)
54 |> Timex.to_naive_datetime()
55 |> NaiveDateTime.truncate(:second)
59 %{id: note_remote_public_id} =
62 |> Ecto.Changeset.change(%{updated_at: date})
65 note_remote_non_public =
66 %{id: note_remote_non_public_id, data: note_remote_non_public_data} =
70 note_remote_non_public
71 |> Ecto.Changeset.change(%{
73 data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
77 assert length(Repo.all(Object)) == 3
79 Mix.Tasks.Pleroma.Database.run(["prune_objects"])
81 assert length(Repo.all(Object)) == 1
82 refute Object.get_by_id(note_remote_public_id)
83 refute Object.get_by_id(note_remote_non_public_id)
86 test "with the --keep-non-public option it still keeps non-public posts even if they are not local" do
87 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
91 |> Timex.shift(days: -deadline)
92 |> Timex.to_naive_datetime()
93 |> NaiveDateTime.truncate(:second)
97 %{id: note_remote_id} =
100 |> Ecto.Changeset.change(%{updated_at: date})
103 note_remote_non_public =
104 %{data: note_remote_non_public_data} =
108 note_remote_non_public
109 |> Ecto.Changeset.change(%{
111 data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
115 assert length(Repo.all(Object)) == 3
117 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-non-public"])
119 assert length(Repo.all(Object)) == 2
120 refute Object.get_by_id(note_remote_id)
123 test "with the --keep-threads and --keep-non-public option it keeps old threads with non-public replies even if the interaction is not local" do
124 # For non-public we only check Create Activities because only these are relevant for threads
125 # Flags are always non-public, Announces from relays can be non-public...
126 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
130 |> Timex.shift(days: -deadline)
131 |> Timex.to_naive_datetime()
132 |> NaiveDateTime.truncate(:second)
134 remote_user1 = insert(:user, local: false)
135 remote_user2 = insert(:user, local: false)
137 # Old remote non-public reply (should be kept)
138 {:ok, old_remote_post1_activity} =
139 CommonAPI.post(remote_user1, %{status: "some thing", local: false})
141 old_remote_post1_activity
142 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
145 {:ok, old_remote_non_public_reply_activity} =
146 CommonAPI.post(remote_user2, %{
147 status: "some reply",
148 in_reply_to_status_id: old_remote_post1_activity.id
151 old_remote_non_public_reply_activity
152 |> Ecto.Changeset.change(%{
154 updated_at: old_insert_date,
155 data: old_remote_non_public_reply_activity.data |> update_in(["to"], fn _ -> [] end)
159 # Old remote non-public Announce (should be removed)
160 {:ok, old_remote_post2_activity = %{data: %{"object" => old_remote_post2_id}}} =
161 CommonAPI.post(remote_user1, %{status: "some thing", local: false})
163 old_remote_post2_activity
164 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
167 {:ok, old_remote_non_public_repeat_activity} =
168 CommonAPI.repeat(old_remote_post2_activity.id, remote_user2)
170 old_remote_non_public_repeat_activity
171 |> Ecto.Changeset.change(%{
173 updated_at: old_insert_date,
174 data: old_remote_non_public_repeat_activity.data |> update_in(["to"], fn _ -> [] end)
178 assert length(Repo.all(Object)) == 3
180 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads", "--keep-non-public"])
182 Repo.all(Pleroma.Activity)
183 assert length(Repo.all(Object)) == 2
184 refute Object.get_by_ap_id(old_remote_post2_id)
187 test "with the --keep-threads option it still keeps non-old threads even with no local interactions" do
188 remote_user = insert(:user, local: false)
189 remote_user2 = insert(:user, local: false)
191 {:ok, remote_post_activity} =
192 CommonAPI.post(remote_user, %{status: "some thing", local: false})
194 {:ok, remote_post_reply_activity} =
195 CommonAPI.post(remote_user2, %{
196 status: "some reply",
197 in_reply_to_status_id: remote_post_activity.id
201 |> Ecto.Changeset.change(%{local: false})
204 remote_post_reply_activity
205 |> Ecto.Changeset.change(%{local: false})
208 assert length(Repo.all(Object)) == 2
210 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
212 assert length(Repo.all(Object)) == 2
215 test "with the --keep-threads option it deletes old threads with no local interaction" do
216 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
220 |> Timex.shift(days: -deadline)
221 |> Timex.to_naive_datetime()
222 |> NaiveDateTime.truncate(:second)
224 remote_user = insert(:user, local: false)
225 remote_user2 = insert(:user, local: false)
227 {:ok, old_remote_post_activity} =
228 CommonAPI.post(remote_user, %{status: "some thing", local: false})
230 old_remote_post_activity
231 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
234 {:ok, old_remote_post_reply_activity} =
235 CommonAPI.post(remote_user2, %{
236 status: "some reply",
237 in_reply_to_status_id: old_remote_post_activity.id
240 old_remote_post_reply_activity
241 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
244 {:ok, old_favourite_activity} =
245 CommonAPI.favorite(remote_user2, old_remote_post_activity.id)
247 old_favourite_activity
248 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
251 {:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post_activity.id, remote_user2)
254 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
257 assert length(Repo.all(Object)) == 2
259 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
261 assert length(Repo.all(Object)) == 0
264 test "with the --keep-threads option it keeps old threads with local interaction" do
265 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
269 |> Timex.shift(days: -deadline)
270 |> Timex.to_naive_datetime()
271 |> NaiveDateTime.truncate(:second)
273 remote_user = insert(:user, local: false)
274 local_user = insert(:user, local: true)
277 {:ok, old_remote_post1_activity} =
278 CommonAPI.post(remote_user, %{status: "some thing", local: false})
280 old_remote_post1_activity
281 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
284 {:ok, old_local_post2_reply_activity} =
285 CommonAPI.post(local_user, %{
286 status: "some reply",
287 in_reply_to_status_id: old_remote_post1_activity.id
290 old_local_post2_reply_activity
291 |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
295 {:ok, old_remote_post3_activity} =
296 CommonAPI.post(remote_user, %{status: "some thing", local: false})
298 old_remote_post3_activity
299 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
302 {:ok, old_favourite_activity} = CommonAPI.favorite(local_user, old_remote_post3_activity.id)
304 old_favourite_activity
305 |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
309 {:ok, old_remote_post4_activity} =
310 CommonAPI.post(remote_user, %{status: "some thing", local: false})
312 old_remote_post4_activity
313 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
316 {:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post4_activity.id, local_user)
319 |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
322 assert length(Repo.all(Object)) == 4
324 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
326 assert length(Repo.all(Object)) == 4
329 test "with the --keep-threads option it keeps old threads with bookmarked posts" do
330 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
334 |> Timex.shift(days: -deadline)
335 |> Timex.to_naive_datetime()
336 |> NaiveDateTime.truncate(:second)
338 remote_user = insert(:user, local: false)
339 local_user = insert(:user, local: true)
341 {:ok, old_remote_post_activity} =
342 CommonAPI.post(remote_user, %{status: "some thing", local: false})
344 old_remote_post_activity
345 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
348 Pleroma.Bookmark.create(local_user.id, old_remote_post_activity.id)
350 assert length(Repo.all(Object)) == 1
352 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
354 assert length(Repo.all(Object)) == 1
357 test "We don't have unexpected tables which may contain objects that are referenced by activities" do
358 # We can delete orphaned activities. For that we look for the objects they reference in the 'objects', 'activities', and 'users' table.
359 # If someone adds another table with objects (idk, maybe with separate relations, or collections or w/e), then we need to make sure we
360 # add logic for that in the 'prune_objects' task so that we don't wrongly delete their corresponding activities.
361 # So when someone adds (or removes) a table, this test will fail.
362 # Either the table contains objects which can be referenced from the activities table
363 # => in that case the prune_objects job should be adapted so we don't delete activities who still have the referenced object.
364 # Or it doesn't contain objects which can be referenced from the activities table
365 # => in that case you can add/remove the table to/from this (sorted) list.
368 "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';"
372 ["announcement_read_relationships"],
377 ["chat_message_references"],
380 ["conversation_participation_recipient_ships"],
381 ["conversation_participations"],
384 ["data_migration_failed_ids"],
388 ["following_relationships"],
390 ["hashtags_objects"],
397 ["oauth_authorizations"],
402 ["password_reset_tokens"],
403 ["push_subscriptions"],
406 ["scheduled_activities"],
407 ["schema_migrations"],
409 ["user_follows_hashtag"],
410 ["user_frontend_setting_profiles"],
411 ["user_invite_tokens"],
413 ["user_relationships"],
418 test "it prunes orphaned activities with the --prune-orphaned-activities" do
419 # Add a remote activity which references an Object
420 %Object{} |> Map.merge(%{data: %{"id" => "object_for_activity"}}) |> Repo.insert()
425 data: %{"id" => "remote_activity_with_object", "object" => "object_for_activity"}
429 # Add a remote activity which references an activity
434 "id" => "remote_activity_with_activity",
435 "object" => "remote_activity_with_object"
440 # Add a remote activity which references an Actor
441 %User{} |> Map.merge(%{ap_id: "actor"}) |> Repo.insert()
446 data: %{"id" => "remote_activity_with_actor", "object" => "actor"}
450 # Add a remote activity without existing referenced object, activity or actor
455 "id" => "remote_activity_without_existing_referenced_object",
456 "object" => "non_existing"
461 # Add a local activity without existing referenced object, activity or actor
465 data: %{"id" => "local_activity_with_actor", "object" => "non_existing"}
469 # The remote activities without existing reference, and only the remote activities without existing reference, are deleted
470 # if, and only if, we provide the --prune-orphaned-activities option
471 assert length(Repo.all(Activity)) == 5
472 Mix.Tasks.Pleroma.Database.run(["prune_objects"])
473 assert length(Repo.all(Activity)) == 5
474 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
475 activities = Repo.all(Activity)
477 assert "remote_activity_without_existing_referenced_object" not in Enum.map(
479 fn a -> a.data["id"] end
482 assert length(activities) == 4
485 test "it prunes orphaned activities with the --prune-orphaned-activities when the objects are referenced from an array" do
486 %Object{} |> Map.merge(%{data: %{"id" => "existing_object"}}) |> Repo.insert()
487 %User{} |> Map.merge(%{ap_id: "existing_actor"}) |> Repo.insert()
489 # Multiple objects, one object exists (keep)
494 "id" => "remote_activity_existing_object",
495 "object" => ["non_ existing_object", "existing_object"]
500 # Multiple objects, one actor exists (keep)
505 "id" => "remote_activity_existing_actor",
506 "object" => ["non_ existing_object", "existing_actor"]
511 # Multiple objects, one activity exists (keep)
516 "id" => "remote_activity_existing_activity",
517 "object" => ["non_ existing_object", "remote_activity_existing_actor"]
522 # Multiple objects none exist (prune)
527 "id" => "remote_activity_without_existing_referenced_object",
528 "object" => ["owo", "whats_this"]
533 assert length(Repo.all(Activity)) == 4
534 Mix.Tasks.Pleroma.Database.run(["prune_objects"])
535 assert length(Repo.all(Activity)) == 4
536 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
537 activities = Repo.all(Activity)
538 assert length(activities) == 3
540 assert "remote_activity_without_existing_referenced_object" not in Enum.map(
542 fn a -> a.data["id"] end
545 assert length(activities) == 3
549 describe "running update_users_following_followers_counts" do
550 test "following and followers count are updated" do
551 [user, user2] = insert_pair(:user)
552 {:ok, %User{} = user, _user2} = User.follow(user, user2)
554 following = User.following(user)
556 assert length(following) == 2
557 assert user.follower_count == 0
561 |> Ecto.Changeset.change(%{follower_count: 3})
564 assert user.follower_count == 3
567 Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"])
569 user = User.get_by_id(user.id)
571 assert length(User.following(user)) == 2
572 assert user.follower_count == 0
576 describe "running fix_likes_collections" do
577 test "it turns OrderedCollection likes into empty arrays" do
578 [user, user2] = insert_pair(:user)
580 {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{status: "test"})
581 {:ok, %{object: object2}} = CommonAPI.post(user, %{status: "test test"})
583 CommonAPI.favorite(user2, id)
587 "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
588 "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
590 "type" => "OrderedCollection"
593 new_data = Map.put(object2.data, "likes", likes)
596 |> Ecto.Changeset.change(%{data: new_data})
599 assert length(Object.get_by_id(object.id).data["likes"]) == 1
600 assert is_map(Object.get_by_id(object2.id).data["likes"])
602 assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"])
604 assert length(Object.get_by_id(object.id).data["likes"]) == 1
605 assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
609 describe "ensure_expiration" do
610 test "it adds to expiration old statuses" do
611 activity1 = insert(:note_activity)
613 {:ok, inserted_at, 0} = DateTime.from_iso8601("2015-01-23T23:50:07Z")
614 activity2 = insert(:note_activity, %{inserted_at: inserted_at})
616 %{id: activity_id3} = insert(:note_activity)
618 expires_at = DateTime.add(DateTime.utc_now(), 60 * 61)
620 Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
621 activity_id: activity_id3,
622 expires_at: expires_at
625 Mix.Tasks.Pleroma.Database.run(["ensure_expiration"])
628 worker: Pleroma.Workers.PurgeExpiredActivity,
629 args: %{activity_id: activity1.id},
631 activity1.inserted_at
632 |> DateTime.from_naive!("Etc/UTC")
633 |> Timex.shift(days: 365)
637 worker: Pleroma.Workers.PurgeExpiredActivity,
638 args: %{activity_id: activity2.id},
640 activity2.inserted_at
641 |> DateTime.from_naive!("Etc/UTC")
642 |> Timex.shift(days: 365)
646 worker: Pleroma.Workers.PurgeExpiredActivity,
647 args: %{activity_id: activity_id3},
648 scheduled_at: expires_at