1 # Pleroma: A lightweight social networking server
2 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
3 # SPDX-License-Identifier: AGPL-3.0-only
5 defmodule Mix.Tasks.Pleroma.DatabaseTest do
6 use Pleroma.DataCase, async: true
7 use Oban.Testing, repo: Pleroma.Repo
13 alias Pleroma.Web.CommonAPI
15 import Pleroma.Factory
18 Mix.shell(Mix.Shell.Process)
21 Mix.shell(Mix.Shell.IO)
27 describe "running remove_embedded_objects" do
28 test "it replaces objects with references" do
30 {:ok, activity} = CommonAPI.post(user, %{status: "test"})
31 new_data = Map.put(activity.data, "object", activity.object.data)
35 |> Activity.change(%{data: new_data})
38 assert is_map(activity.data["object"])
40 Mix.Tasks.Pleroma.Database.run(["remove_embedded_objects"])
42 activity = Activity.get_by_id_with_object(activity.id)
43 assert is_binary(activity.data["object"])
47 describe "prune_objects" do
48 test "it prunes old objects from the database" do
49 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
53 |> Timex.shift(days: -deadline)
54 |> Timex.to_naive_datetime()
55 |> NaiveDateTime.truncate(:second)
59 %{id: note_remote_public_id} =
62 |> Ecto.Changeset.change(%{updated_at: date})
65 note_remote_non_public =
66 %{id: note_remote_non_public_id, data: note_remote_non_public_data} =
70 note_remote_non_public
71 |> Ecto.Changeset.change(%{
73 data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
77 assert length(Repo.all(Object)) == 3
79 Mix.Tasks.Pleroma.Database.run(["prune_objects"])
81 assert length(Repo.all(Object)) == 1
82 refute Object.get_by_id(note_remote_public_id)
83 refute Object.get_by_id(note_remote_non_public_id)
86 test "with the --keep-non-public option it still keeps non-public posts even if they are not local" do
87 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
91 |> Timex.shift(days: -deadline)
92 |> Timex.to_naive_datetime()
93 |> NaiveDateTime.truncate(:second)
97 %{id: note_remote_id} =
100 |> Ecto.Changeset.change(%{updated_at: date})
103 note_remote_non_public =
104 %{data: note_remote_non_public_data} =
108 note_remote_non_public
109 |> Ecto.Changeset.change(%{
111 data: note_remote_non_public_data |> update_in(["to"], fn _ -> [] end)
115 assert length(Repo.all(Object)) == 3
117 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-non-public"])
119 assert length(Repo.all(Object)) == 2
120 refute Object.get_by_id(note_remote_id)
123 test "with the --keep-threads and --keep-non-public option it keeps old threads with non-public replies even if the interaction is not local" do
124 # For non-public we only check Create Activities because only these are relevant for threads
125 # Flags are always non-public, Announces from relays can be non-public...
126 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
130 |> Timex.shift(days: -deadline)
131 |> Timex.to_naive_datetime()
132 |> NaiveDateTime.truncate(:second)
134 remote_user1 = insert(:user, local: false)
135 remote_user2 = insert(:user, local: false)
137 # Old remote non-public reply (should be kept)
138 {:ok, old_remote_post1_activity} =
139 CommonAPI.post(remote_user1, %{status: "some thing", local: false})
141 old_remote_post1_activity
142 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
145 {:ok, old_remote_non_public_reply_activity} =
146 CommonAPI.post(remote_user2, %{
147 status: "some reply",
148 in_reply_to_status_id: old_remote_post1_activity.id
151 old_remote_non_public_reply_activity
152 |> Ecto.Changeset.change(%{
154 updated_at: old_insert_date,
155 data: old_remote_non_public_reply_activity.data |> update_in(["to"], fn _ -> [] end)
159 # Old remote non-public Announce (should be removed)
160 {:ok, old_remote_post2_activity = %{data: %{"object" => old_remote_post2_id}}} =
161 CommonAPI.post(remote_user1, %{status: "some thing", local: false})
163 old_remote_post2_activity
164 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
167 {:ok, old_remote_non_public_repeat_activity} =
168 CommonAPI.repeat(old_remote_post2_activity.id, remote_user2)
170 old_remote_non_public_repeat_activity
171 |> Ecto.Changeset.change(%{
173 updated_at: old_insert_date,
174 data: old_remote_non_public_repeat_activity.data |> update_in(["to"], fn _ -> [] end)
178 assert length(Repo.all(Object)) == 3
180 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads", "--keep-non-public"])
182 Repo.all(Pleroma.Activity)
183 assert length(Repo.all(Object)) == 2
184 refute Object.get_by_ap_id(old_remote_post2_id)
187 test "with the --keep-threads option it still keeps non-old threads even with no local interactions" do
188 remote_user = insert(:user, local: false)
189 remote_user2 = insert(:user, local: false)
191 {:ok, remote_post_activity} =
192 CommonAPI.post(remote_user, %{status: "some thing", local: false})
194 {:ok, remote_post_reply_activity} =
195 CommonAPI.post(remote_user2, %{
196 status: "some reply",
197 in_reply_to_status_id: remote_post_activity.id
201 |> Ecto.Changeset.change(%{local: false})
204 remote_post_reply_activity
205 |> Ecto.Changeset.change(%{local: false})
208 assert length(Repo.all(Object)) == 2
210 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
212 assert length(Repo.all(Object)) == 2
215 test "with the --keep-threads option it deletes old threads with no local interaction" do
216 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
220 |> Timex.shift(days: -deadline)
221 |> Timex.to_naive_datetime()
222 |> NaiveDateTime.truncate(:second)
224 remote_user = insert(:user, local: false)
225 remote_user2 = insert(:user, local: false)
227 {:ok, old_remote_post_activity} =
228 CommonAPI.post(remote_user, %{status: "some thing", local: false})
230 old_remote_post_activity
231 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
234 {:ok, old_remote_post_reply_activity} =
235 CommonAPI.post(remote_user2, %{
236 status: "some reply",
237 in_reply_to_status_id: old_remote_post_activity.id
240 old_remote_post_reply_activity
241 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
244 {:ok, old_favourite_activity} =
245 CommonAPI.favorite(remote_user2, old_remote_post_activity.id)
247 old_favourite_activity
248 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
251 {:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post_activity.id, remote_user2)
254 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
257 assert length(Repo.all(Object)) == 2
259 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
261 assert length(Repo.all(Object)) == 0
264 test "with the --keep-threads option it keeps old threads with local interaction" do
265 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
269 |> Timex.shift(days: -deadline)
270 |> Timex.to_naive_datetime()
271 |> NaiveDateTime.truncate(:second)
273 remote_user = insert(:user, local: false)
274 local_user = insert(:user, local: true)
277 {:ok, old_remote_post1_activity} =
278 CommonAPI.post(remote_user, %{status: "some thing", local: false})
280 old_remote_post1_activity
281 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
284 {:ok, old_local_post2_reply_activity} =
285 CommonAPI.post(local_user, %{
286 status: "some reply",
287 in_reply_to_status_id: old_remote_post1_activity.id
290 old_local_post2_reply_activity
291 |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
295 {:ok, old_remote_post3_activity} =
296 CommonAPI.post(remote_user, %{status: "some thing", local: false})
298 old_remote_post3_activity
299 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
302 {:ok, old_favourite_activity} = CommonAPI.favorite(local_user, old_remote_post3_activity.id)
304 old_favourite_activity
305 |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
309 {:ok, old_remote_post4_activity} =
310 CommonAPI.post(remote_user, %{status: "some thing", local: false})
312 old_remote_post4_activity
313 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
316 {:ok, old_repeat_activity} = CommonAPI.repeat(old_remote_post4_activity.id, local_user)
319 |> Ecto.Changeset.change(%{local: true, updated_at: old_insert_date})
322 assert length(Repo.all(Object)) == 4
324 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
326 assert length(Repo.all(Object)) == 4
329 test "with the --keep-threads option it keeps old threads with bookmarked posts" do
330 deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + 1
334 |> Timex.shift(days: -deadline)
335 |> Timex.to_naive_datetime()
336 |> NaiveDateTime.truncate(:second)
338 remote_user = insert(:user, local: false)
339 local_user = insert(:user, local: true)
341 {:ok, old_remote_post_activity} =
342 CommonAPI.post(remote_user, %{status: "some thing", local: false})
344 old_remote_post_activity
345 |> Ecto.Changeset.change(%{local: false, updated_at: old_insert_date})
348 Pleroma.Bookmark.create(local_user.id, old_remote_post_activity.id)
350 assert length(Repo.all(Object)) == 1
352 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--keep-threads"])
354 assert length(Repo.all(Object)) == 1
357 test "We don't have unexpected tables which may contain objects that are referenced by activities" do
358 # We can delete orphaned activities. For that we look for the objects they reference in the 'objects', 'activities', and 'users' table.
359 # If someone adds another table with objects (idk, maybe with separate relations, or collections or w/e), then we need to make sure we
360 # add logic for that in the 'prune_objects' task so that we don't wrongly delete their corresponding activities.
361 # So when someone adds (or removes) a table, this test will fail.
362 # Either the table contains objects which can be referenced from the activities table
363 # => in that case the prune_objects job should be adapted so we don't delete activities who still have the referenced object.
364 # Or it doesn't contain objects which can be referenced from the activities table
365 # => in that case you can add/remove the table to/from this (sorted) list.
368 "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';"
372 ["announcement_read_relationships"],
377 ["chat_message_references"],
380 ["conversation_participation_recipient_ships"],
381 ["conversation_participations"],
384 ["data_migration_failed_ids"],
388 ["following_relationships"],
390 ["hashtags_objects"],
397 ["oauth_authorizations"],
402 ["password_reset_tokens"],
403 ["push_subscriptions"],
406 ["scheduled_activities"],
407 ["schema_migrations"],
409 ["user_follows_hashtag"],
410 ["user_frontend_setting_profiles"],
411 ["user_invite_tokens"],
413 ["user_relationships"],
418 test "it prunes orphaned activities with the --prune-orphaned-activities" do
419 %Object{} |> Map.merge(%{data: %{"id" => "object_for_activity"}}) |> Repo.insert()
424 data: %{"id" => "remote_activity_with_object", "object" => "object_for_activity"}
432 "id" => "remote_activity_with_activity",
433 "object" => "remote_activity_with_object"
438 %User{} |> Map.merge(%{ap_id: "actor"}) |> Repo.insert()
443 data: %{"id" => "remote_activity_with_actor", "object" => "actor"}
451 "id" => "remote_activity_without_existing_referenced_object",
452 "object" => "non_existing"
460 data: %{"id" => "local_activity_with_actor", "object" => "non_existing"}
464 assert length(Repo.all(Activity)) == 5
465 Mix.Tasks.Pleroma.Database.run(["prune_objects"])
466 assert length(Repo.all(Activity)) == 5
467 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
468 activities = Repo.all(Activity)
470 assert "remote_activity_without_existing_referenced_object" not in Enum.map(
472 fn a -> a.data["id"] end
475 assert length(activities) == 4
478 test "it prunes orphaned activities with the --prune-orphaned-activities when the objects are referenced from an array" do
479 %Object{} |> Map.merge(%{data: %{"id" => "existing_object"}}) |> Repo.insert()
480 %User{} |> Map.merge(%{ap_id: "existing_actor"}) |> Repo.insert()
486 "id" => "remote_activity_existing_object",
487 "object" => ["non_ existing_object", "existing_object"]
496 "id" => "remote_activity_existing_actor",
497 "object" => ["non_ existing_object", "existing_actor"]
506 "id" => "remote_activity_existing_activity",
507 "object" => ["non_ existing_object", "remote_activity_existing_actor"]
516 "id" => "remote_activity_without_existing_referenced_object",
517 "object" => ["owo", "whats_this"]
522 assert length(Repo.all(Activity)) == 4
523 Mix.Tasks.Pleroma.Database.run(["prune_objects"])
524 assert length(Repo.all(Activity)) == 4
525 Mix.Tasks.Pleroma.Database.run(["prune_objects", "--prune-orphaned-activities"])
526 activities = Repo.all(Activity)
527 assert length(activities) == 3
529 assert "remote_activity_without_existing_referenced_object" not in Enum.map(
531 fn a -> a.data["id"] end
534 assert length(activities) == 3
538 describe "running update_users_following_followers_counts" do
539 test "following and followers count are updated" do
540 [user, user2] = insert_pair(:user)
541 {:ok, %User{} = user, _user2} = User.follow(user, user2)
543 following = User.following(user)
545 assert length(following) == 2
546 assert user.follower_count == 0
550 |> Ecto.Changeset.change(%{follower_count: 3})
553 assert user.follower_count == 3
556 Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"])
558 user = User.get_by_id(user.id)
560 assert length(User.following(user)) == 2
561 assert user.follower_count == 0
565 describe "running fix_likes_collections" do
566 test "it turns OrderedCollection likes into empty arrays" do
567 [user, user2] = insert_pair(:user)
569 {:ok, %{id: id, object: object}} = CommonAPI.post(user, %{status: "test"})
570 {:ok, %{object: object2}} = CommonAPI.post(user, %{status: "test test"})
572 CommonAPI.favorite(user2, id)
576 "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1",
577 "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes",
579 "type" => "OrderedCollection"
582 new_data = Map.put(object2.data, "likes", likes)
585 |> Ecto.Changeset.change(%{data: new_data})
588 assert length(Object.get_by_id(object.id).data["likes"]) == 1
589 assert is_map(Object.get_by_id(object2.id).data["likes"])
591 assert :ok == Mix.Tasks.Pleroma.Database.run(["fix_likes_collections"])
593 assert length(Object.get_by_id(object.id).data["likes"]) == 1
594 assert Enum.empty?(Object.get_by_id(object2.id).data["likes"])
598 describe "ensure_expiration" do
599 test "it adds to expiration old statuses" do
600 activity1 = insert(:note_activity)
602 {:ok, inserted_at, 0} = DateTime.from_iso8601("2015-01-23T23:50:07Z")
603 activity2 = insert(:note_activity, %{inserted_at: inserted_at})
605 %{id: activity_id3} = insert(:note_activity)
607 expires_at = DateTime.add(DateTime.utc_now(), 60 * 61)
609 Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
610 activity_id: activity_id3,
611 expires_at: expires_at
614 Mix.Tasks.Pleroma.Database.run(["ensure_expiration"])
617 worker: Pleroma.Workers.PurgeExpiredActivity,
618 args: %{activity_id: activity1.id},
620 activity1.inserted_at
621 |> DateTime.from_naive!("Etc/UTC")
622 |> Timex.shift(days: 365)
626 worker: Pleroma.Workers.PurgeExpiredActivity,
627 args: %{activity_id: activity2.id},
629 activity2.inserted_at
630 |> DateTime.from_naive!("Etc/UTC")
631 |> Timex.shift(days: 365)
635 worker: Pleroma.Workers.PurgeExpiredActivity,
636 args: %{activity_id: activity_id3},
637 scheduled_at: expires_at