[#3213] `rescue` around potentially-raising `Repo.insert_all/_` calls. Misc. improvem...

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 23567a97c8a839d2854c6c6651a535da6df127a8..a7b5f6ac055257bd57524952e5084e340687d65e 100644 (file)
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,7 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: Reports now ordered by newest
 
 </details>
-- Extracted object hashtags into separate table in order to improve hashtag timeline performance (via background migration in `Pleroma.Migrators.HashtagsTableMigrator`). 
+- Improved hashtag timeline performance (requires a background migration). 
 
 ### Added
 

diff --git a/config/config.exs b/config/config.exs
index 8a7c466d3fe4a67880bc53603e5788b254f9a579..0fbca06f3f8f8d2cd35c0cdef8a39c0fb378efea 100644 (file)
--- a/config/config.exs
+++ b/config/config.exs
@@ -556,7 +556,6 @@ config :pleroma, Oban,
     remote_fetcher: 2,
     attachments_cleanup: 1,
     new_users_digest: 1,
-    hashtags_cleanup: 1,
     mute_expire: 5
   ],
   plugins: [Oban.Plugins.Pruner],

diff --git a/config/description.exs b/config/description.exs
index 2e96024f5998058b2120d6f04a2429ace9536c1d..29fc5fbd48d88cdaa1c26162cbf56c9b2abe9d40 100644 (file)
--- a/config/description.exs
+++ b/config/description.exs
@@ -473,6 +473,20 @@ config :pleroma, :config_description, [
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :populate_hashtags_table,
+    type: :group,
+    description: "`populate_hashtags_table` background migration settings",
+    children: [
+      %{
+        key: :sleep_interval_ms,
+        type: :integer,
+        description:
+          "Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :instance,

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index ad57684656f3f1422bc0a7017f49249d3d2c32e7..68a5a3c7fcc7229cbc75dd00dca28be5bb605c59 100644 (file)
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -65,6 +65,12 @@ To add configuration to your config file, you can copy it from the base config.
 * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
 * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
 
+## :database
+* `improved_hashtag_timeline`: If `true`, hashtags will be fetched from `hashtags` table for hashtags timeline. When `false`, object-embedded hashtags will be used (slower). Is auto-set to `true` (unless overridden) when `HashtagsTableMigrator` completes.
+
+## Background migrations
+* `populate_hashtags_table/sleep_interval_ms`: Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances).
+
 ## Welcome
 * `direct_message`: - welcome message sent as a direct message.
   * `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.

diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex
index de52c4dae66b5a98e744ea7d4db259e98ec5921f..0d6a4d09e8caa5f8bd16cd5116ce7bd165b1854e 100644 (file)
--- a/lib/pleroma/hashtag.ex
+++ b/lib/pleroma/hashtag.ex
@@ -47,16 +47,20 @@ defmodule Pleroma.Hashtag do
         |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
       end)
 
-    with {:ok, %{query_op: hashtags}} <-
-           Multi.new()
-           |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing)
-           |> Multi.run(:query_op, fn _repo, _changes ->
-             {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
-           end)
-           |> Repo.transaction() do
-      {:ok, hashtags}
-    else
-      {:error, _name, value, _changes_so_far} -> {:error, value}
+    try do
+      with {:ok, %{query_op: hashtags}} <-
+             Multi.new()
+             |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing)
+             |> Multi.run(:query_op, fn _repo, _changes ->
+               {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
+             end)
+             |> Repo.transaction() do
+        {:ok, hashtags}
+      else
+        {:error, _name, value, _changes_so_far} -> {:error, value}
+      end
+    rescue
+      e -> {:error, e}
     end
   end
 
@@ -74,8 +78,9 @@ defmodule Pleroma.Hashtag do
              where: hto.object_id == ^object_id,
              select: hto.hashtag_id
            )
-           |> Repo.delete_all() do
-      delete_unreferenced(hashtag_ids)
+           |> Repo.delete_all(),
+         {:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
+      {:ok, length(hashtag_ids), unreferenced_count}
     end
   end
 

diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex
index c53f6be127188104a1d8a510186a98bcbf317f9c..432c3401a1b4ccc72037fdb40658a591a9d0bc49 100644 (file)
--- a/lib/pleroma/migrators/hashtags_table_migrator.ex
+++ b/lib/pleroma/migrators/hashtags_table_migrator.ex
@@ -214,15 +214,20 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
         maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
         expected_rows = length(hashtag_records)
 
-        with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do
-          object.id
-        else
+        base_error =
+          "ERROR when inserting #{expected_rows} hashtags_objects for obj. #{object.id}"
+
+        try do
+          with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do
+            object.id
+          else
+            e ->
+              Logger.error("#{base_error}: #{inspect(e)}")
+              Repo.rollback(object.id)
+          end
+        rescue
           e ->
-            error =
-              "ERROR when inserting #{expected_rows} hashtags_objects " <>
-                "for object #{object.id}: #{inspect(e)}"
-
-            Logger.error(error)
+            Logger.error("#{base_error}: #{inspect(e)}")
             Repo.rollback(object.id)
         end
       else