From: floatingghost Date: Thu, 1 Dec 2022 15:00:53 +0000 (+0000) Subject: Fixing up deletes a bit (#327) X-Git-Url: https://git.squeep.com/?a=commitdiff_plain;h=db60640c5b91cd1ce2756565835673fa57afe082;p=akkoma Fixing up deletes a bit (#327) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/327 --- diff --git a/CHANGELOG.md b/CHANGELOG.md index b49f77428..136c7e65f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Config: HTTP timeout options, :pool\_timeout and :receive\_timeout - Added statistic gathering about instances which do/don't have signed fetches when they request from us - Ability to set a default post expiry time, after which the post will be deleted. If used in concert with ActivityExpiration MRF, the expiry which comes _sooner_ will be applied. +- Regular task to prune local transient activities +- Task to manually run the transient prune job (pleroma.database prune\_task) ## Changed - MastoAPI: Accept BooleanLike input on `/api/v1/accounts/:id/follow` (fixes follows with mastodon.py) - Relays from akkoma are now off by default - NormalizeMarkup MRF is now on by default - Follow/Block/Mute imports now spin off into *n* tasks to avoid the oban timeout +- Transient activities recieved from remote servers are no longer persisted in the database + +## Upgrade Notes +- If you have an old instance, you will probably want to run `mix pleroma.database prune_task` in the foreground to catch it up with the history of your instance. ## 2022.11 diff --git a/config/config.exs b/config/config.exs index c98a18d39..8e0104400 100644 --- a/config/config.exs +++ b/config/config.exs @@ -569,7 +569,8 @@ config :pleroma, Oban, new_users_digest: 1, mute_expire: 5, search_indexing: 10, - nodeinfo_fetcher: 1 + nodeinfo_fetcher: 1, + database_prune: 1 ], plugins: [ Oban.Plugins.Pruner, @@ -577,7 +578,8 @@ config :pleroma, Oban, ], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, - {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} + {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}, + {"0 3 * * *", Pleroma.Workers.Cron.PruneDatabaseWorker} ] config :pleroma, :workers, @@ -605,7 +607,8 @@ config :pleroma, :workers, new_users_digest: :timer.seconds(10), mute_expire: :timer.seconds(5), search_indexing: :timer.seconds(5), - nodeinfo_fetcher: :timer.seconds(10) + nodeinfo_fetcher: :timer.seconds(10), + database_prune: :timer.minutes(10) ] config :pleroma, Pleroma.Formatter, diff --git a/docs/docs/administration/CLI_tasks/database.md b/docs/docs/administration/CLI_tasks/database.md index 8b2ab93e6..73419dc81 100644 --- a/docs/docs/administration/CLI_tasks/database.md +++ b/docs/docs/administration/CLI_tasks/database.md @@ -159,3 +159,23 @@ Change `default_text_search_config` for database and (if necessary) text_search_ ``` See [PostgreSQL documentation](https://www.postgresql.org/docs/current/textsearch-configuration.html) and `docs/configuration/howto_search_cjk.md` for more detail. + +## Pruning old activities + +Over time, transient `Delete` activities and `Tombstone` objects +can accumulate in your database, inflating its size. This is not ideal. +There is a periodic task to prune these transient objects, +but on first run this may take a while on older instances to catch up +to the current day. + +=== "OTP" + + ```sh + ./bin/pleroma_ctl database prune_task + ``` + +=== "From Source" + + ```sh + mix pleroma.database prune_task + ``` \ No newline at end of file diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 99897e83e..0881974ee 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -110,6 +110,14 @@ defmodule Mix.Tasks.Pleroma.Database do end end + def run(["prune_task"]) do + start_pleroma() + + nil + |> Pleroma.Workers.Cron.PruneDatabaseWorker.perform() + |> IO.inspect() + end + def run(["fix_likes_collections"]) do start_pleroma() diff --git a/lib/pleroma/activity/pruner.ex b/lib/pleroma/activity/pruner.ex new file mode 100644 index 000000000..054ee514a --- /dev/null +++ b/lib/pleroma/activity/pruner.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.Activity.Pruner do + @moduledoc """ + Prunes activities from the database. + """ + @cutoff 30 + + alias Pleroma.Activity + alias Pleroma.Repo + import Ecto.Query + + def prune_deletes do + before_time = cutoff() + + from(a in Activity, + where: fragment("?->>'type' = ?", a.data, "Delete") and a.inserted_at < ^before_time + ) + |> Repo.delete_all(timeout: :infinity) + end + + def prune_undos do + before_time = cutoff() + + from(a in Activity, + where: fragment("?->>'type' = ?", a.data, "Undo") and a.inserted_at < ^before_time + ) + |> Repo.delete_all(timeout: :infinity) + end + + def prune_removes do + before_time = cutoff() + + from(a in Activity, + where: fragment("?->>'type' = ?", a.data, "Remove") and a.inserted_at < ^before_time + ) + |> Repo.delete_all(timeout: :infinity) + end + + defp cutoff do + DateTime.utc_now() |> Timex.shift(days: -@cutoff) + end +end diff --git a/lib/pleroma/object/pruner.ex b/lib/pleroma/object/pruner.ex new file mode 100644 index 000000000..991d8b0eb --- /dev/null +++ b/lib/pleroma/object/pruner.ex @@ -0,0 +1,31 @@ +defmodule Pleroma.Object.Pruner do + @moduledoc """ + Prunes objects from the database. + """ + @cutoff 30 + + alias Pleroma.Object + alias Pleroma.Delivery + alias Pleroma.Repo + import Ecto.Query + + def prune_tombstoned_deliveries do + from(d in Delivery) + |> join(:inner, [d], o in Object, on: d.object_id == o.id) + |> where([d, o], fragment("?->>'type' = ?", o.data, "Tombstone")) + |> Repo.delete_all(timeout: :infinity) + end + + def prune_tombstones do + before_time = cutoff() + + from(o in Object, + where: fragment("?->>'type' = ?", o.data, "Tombstone") and o.inserted_at < ^before_time + ) + |> Repo.delete_all(timeout: :infinity, on_delete: :delete_all) + end + + defp cutoff do + DateTime.utc_now() |> Timex.shift(days: -@cutoff) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 76b99025b..db5dbc815 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -105,6 +105,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + @unpersisted_activity_types ~w[Undo Delete Remove] + @impl true + def persist(%{"type" => type} = object, [local: false] = meta) + when type in @unpersisted_activity_types do + {:ok, object, meta} + {recipients, _, _} = get_recipients(object) + + unpersisted = %Activity{ + data: object, + local: false, + recipients: recipients, + actor: object["actor"] + } + + {:ok, unpersisted, meta} + end + @impl true def persist(object, meta) do with local <- Keyword.fetch!(meta, :local), diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 18643662e..34617a218 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -288,7 +288,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # Tasks this handles: # - Delete and unpins the create activity - # - Replace object with Tombstone # - Set up notification # - Reduce the user note count # - Reduce the reply count diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 80497a252..a3648c458 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -222,7 +222,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> Maps.put_if_present(:language, Pleroma.Web.Gettext.normalize_locale(params[:language])) |> Maps.put_if_present(:status_ttl_days, params[:status_ttl_days], status_ttl_days_value) - IO.inspect(user_params) # What happens here: # # We want to update the user through the pipeline, but the ActivityPub diff --git a/lib/pleroma/workers/cron/database_prune_worker.ex b/lib/pleroma/workers/cron/database_prune_worker.ex new file mode 100644 index 000000000..99ea2e836 --- /dev/null +++ b/lib/pleroma/workers/cron/database_prune_worker.ex @@ -0,0 +1,32 @@ +defmodule Pleroma.Workers.Cron.PruneDatabaseWorker do + @moduledoc """ + The worker to prune old data from the database. + """ + require Logger + use Oban.Worker, queue: "database_prune" + + alias Pleroma.Activity.Pruner, as: ActivityPruner + alias Pleroma.Object.Pruner, as: ObjectPruner + + @impl Oban.Worker + def perform(_job) do + Logger.info("Pruning old data from the database") + + Logger.info("Pruning old deletes") + ActivityPruner.prune_deletes() + + Logger.info("Pruning old undos") + ActivityPruner.prune_undos() + + Logger.info("Pruning old removes") + ActivityPruner.prune_removes() + + Logger.info("Pruning old tombstone delivery entries") + ObjectPruner.prune_tombstoned_deliveries() + + Logger.info("Pruning old tombstones") + ObjectPruner.prune_tombstones() + + :ok + end +end diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 70a8d42d0..518a44c0a 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -14,11 +14,10 @@ defmodule Pleroma.Workers.SearchIndexingWorker do end def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do - object = Pleroma.Object.get_by_id(object_id) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - search_module.remove_from_index(object) + # Fake the object so we can remove it from the index without having to keep it in the DB + search_module.remove_from_index(%Pleroma.Object{id: object_id}) :ok end diff --git a/priv/repo/migrations/20221129105331_add_notification_activity_id_index.exs b/priv/repo/migrations/20221129105331_add_notification_activity_id_index.exs new file mode 100644 index 000000000..b1eb71f72 --- /dev/null +++ b/priv/repo/migrations/20221129105331_add_notification_activity_id_index.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddNotificationActivityIdIndex do + use Ecto.Migration + + def change do + create(index(:notifications, [:activity_id])) + end +end diff --git a/priv/repo/migrations/20221129110627_add_bookmarks_activity_id_index.exs b/priv/repo/migrations/20221129110627_add_bookmarks_activity_id_index.exs new file mode 100644 index 000000000..f7b7911f8 --- /dev/null +++ b/priv/repo/migrations/20221129110627_add_bookmarks_activity_id_index.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddBookmarksActivityIdIndex do + use Ecto.Migration + + def change do + create(index(:bookmarks, [:activity_id])) + end +end diff --git a/priv/repo/migrations/20221129110727_add_report_notes_activity_id_index.exs b/priv/repo/migrations/20221129110727_add_report_notes_activity_id_index.exs new file mode 100644 index 000000000..dfe74c191 --- /dev/null +++ b/priv/repo/migrations/20221129110727_add_report_notes_activity_id_index.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddReportNotesActivityIdIndex do + use Ecto.Migration + + def change do + create(index(:report_notes, [:activity_id])) + end +end diff --git a/priv/repo/migrations/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs b/priv/repo/migrations/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs new file mode 100644 index 000000000..960554a49 --- /dev/null +++ b/priv/repo/migrations/20221129112022_add_cascade_to_report_notes_on_activity_delete.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.AddCascadeToReportNotesOnActivityDelete do + use Ecto.Migration + + def up do + drop(constraint(:report_notes, "report_notes_activity_id_fkey")) + + alter table(:report_notes) do + modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all)) + end + end + + def down do + drop(constraint(:report_notes, "report_notes_activity_id_fkey")) + + alter table(:report_notes) do + modify(:activity_id, references(:activities, type: :uuid)) + end + end +end diff --git a/priv/static/logo-512.png b/priv/static/logo-512.png new file mode 100755 index 000000000..02d36e7ab Binary files /dev/null and b/priv/static/logo-512.png differ diff --git a/priv/static/logo.svg b/priv/static/logo.svg new file mode 100755 index 000000000..fbd5c6ec1 --- /dev/null +++ b/priv/static/logo.svg @@ -0,0 +1,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/pleroma/activity/pruner_test.exs b/test/pleroma/activity/pruner_test.exs new file mode 100644 index 000000000..312d4f5e4 --- /dev/null +++ b/test/pleroma/activity/pruner_test.exs @@ -0,0 +1,27 @@ +defmodule Pleroma.Activity.PrunerTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Activity + alias Pleroma.Activity.Pruner + + import Pleroma.Factory + + describe "prune_deletes" do + test "it prunes old delete objects" do + user = insert(:user) + + new_delete = insert(:delete_activity, type: "Delete", user: user) + + old_delete = + insert(:delete_activity, + type: "Delete", + user: user, + inserted_at: DateTime.utc_now() |> DateTime.add(-31 * 24, :hour) + ) + + Pruner.prune_deletes() + assert Activity.get_by_id(new_delete.id) + refute Activity.get_by_id(old_delete.id) + end + end +end diff --git a/test/pleroma/object/pruner_test.exs b/test/pleroma/object/pruner_test.exs new file mode 100644 index 000000000..73c574b4b --- /dev/null +++ b/test/pleroma/object/pruner_test.exs @@ -0,0 +1,41 @@ +defmodule Pleroma.Object.PrunerTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Delivery + alias Pleroma.Object + alias Pleroma.Object.Pruner + + import Pleroma.Factory + + describe "prune_deletes" do + test "it prunes old delete objects" do + new_tombstone = insert(:tombstone) + + old_tombstone = + insert(:tombstone, + inserted_at: DateTime.utc_now() |> DateTime.add(-31 * 24, :hour) + ) + + Pruner.prune_tombstones() + assert Object.get_by_id(new_tombstone.id) + refute Object.get_by_id(old_tombstone.id) + end + end + + describe "prune_tombstoned_deliveries" do + test "it prunes old tombstone deliveries" do + user = insert(:user) + + tombstone = insert(:tombstone) + tombstoned = insert(:delivery, object: tombstone, user: user) + + note = insert(:note) + not_tombstoned = insert(:delivery, object: note, user: user) + + Pruner.prune_tombstoned_deliveries() + + refute Repo.get(Delivery, tombstoned.id) + assert Repo.get(Delivery, not_tombstoned.id) + end + end +end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index c7b3334f3..8d39b1076 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do alias Pleroma.Activity alias Pleroma.Builders.ActivityBuilder + alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Config alias Pleroma.Notification alias Pleroma.Object @@ -2613,4 +2614,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew") assert user.name == " " end + + describe "persist/1" do + test "should not persist remote delete activities" do + poster = insert(:user, local: false) + {:ok, post} = CommonAPI.post(poster, %{status: "hhhhhh"}) + + {:ok, delete_data, meta} = Builder.delete(poster, post) + local_opts = Keyword.put(meta, :local, false) + {:ok, act, _meta} = ActivityPub.persist(delete_data, local_opts) + refute act.inserted_at + end + + test "should not persist remote undo activities" do + poster = insert(:user, local: false) + liker = insert(:user, local: false) + {:ok, post} = CommonAPI.post(poster, %{status: "hhhhhh"}) + {:ok, like} = CommonAPI.favorite(liker, post.id) + + {:ok, undo_data, meta} = Builder.undo(liker, like) + local_opts = Keyword.put(meta, :local, false) + {:ok, act, _meta} = ActivityPub.persist(undo_data, local_opts) + refute act.inserted_at + end + end end diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs index 435782d0a..a5403f360 100644 --- a/test/pleroma/web/mastodon_api/update_credentials_test.exs +++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs @@ -226,7 +226,7 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do test "does not allow negative integers other than -1 for TTL", %{conn: conn} do conn = patch(conn, "/api/v1/accounts/update_credentials", %{"status_ttl_days" => "-2"}) - assert user_data = json_response_and_validate_schema(conn, 403) + assert json_response_and_validate_schema(conn, 403) end test "updates the user's AKAs", %{conn: conn} do diff --git a/test/support/factory.ex b/test/support/factory.ex index bd9d7fe42..904987aaf 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -233,7 +233,7 @@ defmodule Pleroma.Factory do %Pleroma.Object{data: Map.merge(data, %{"type" => "Article"})} end - def tombstone_factory do + def tombstone_factory(attrs) do data = %{ "type" => "Tombstone", "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), @@ -244,6 +244,7 @@ defmodule Pleroma.Factory do %Pleroma.Object{ data: data } + |> merge_attributes(attrs) end def question_factory(attrs \\ %{}) do @@ -520,6 +521,33 @@ defmodule Pleroma.Factory do |> Map.merge(attrs) end + def delete_activity_factory(attrs \\ %{}) do + user = attrs[:user] || insert(:user) + note_activity = attrs[:note_activity] || insert(:note_activity, user: user) + + data_attrs = attrs[:data_attrs] || %{} + attrs = Map.drop(attrs, [:user, :data_attrs]) + + data = + %{ + "id" => Pleroma.Web.ActivityPub.Utils.generate_activity_id(), + "type" => "Delete", + "actor" => note_activity.data["actor"], + "to" => note_activity.data["to"], + "object" => note_activity.data["id"], + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "context" => note_activity.data["context"] + } + |> Map.merge(data_attrs) + + %Pleroma.Activity{ + data: data, + actor: data["actor"], + recipients: data["to"] + } + |> Map.merge(attrs) + end + def oauth_app_factory do %Pleroma.Web.OAuth.App{ client_name: sequence(:client_name, &"Some client #{&1}"), @@ -676,4 +704,14 @@ defmodule Pleroma.Factory do } |> Map.merge(params) end + + def delivery_factory(params \\ %{}) do + object = Map.get(params, :object, build(:note)) + user = Map.get(params, :user, build(:user)) + + %Pleroma.Delivery{ + object: object, + user: user + } + end end