Friendship ended with Postgresql now Cachex is my best friend

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 36a3694f240abbe8d011c90ccf0b22645333df3e..4b997c04862bdbacada2b667b73b443e6d2d290d 100644 (file)
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -53,6 +53,16 @@ defmodule Pleroma.Application do
           ],
           id: :cachex_object
         ),
+        worker(
+          Cachex,
+          [
+            :scrubber_cache,
+            [
+              limit: 2500
+            ]
+          ],
+          id: :cachex_scrubber
+        ),
         worker(
           Cachex,
           [

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index cc4a2181a060c5d5c9e965bce9b4cebd34b5e4fc..e148c1d75593254a966a226e70e34fa8f60f380c 100644 (file)
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Object do
   use Ecto.Schema
-  alias Pleroma.{Repo, Object, User, Activity}
+  alias Pleroma.{Repo, Object, User, Activity, HTML}
   import Ecto.{Query, Changeset}
 
   schema "objects" do
@@ -73,4 +73,36 @@ defmodule Pleroma.Object do
       {:ok, object}
     end
   end
+  
+  def get_cached_scrubbed_html(content, scrubbers, object) do
+    key = "#{generate_scrubber_signature(scrubbers)}|#{object.id}"
+    Cachex.fetch!(:scrubber_cache, key, fn(_key) -> ensure_scrubbed_html(content, scrubbers) end )
+  end
+
+  def get_cached_stripped_html(content, object) do
+    get_cached_scrubbed_html(content, HtmlSanitizeEx.Scrubber.StripTags, object)
+  end
+
+  def ensure_scrubbed_html(
+        content,
+        scrubbers
+      ) do
+      {:commit, HTML.filter_tags(content, scrubbers)}
+  end
+  
+  defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
+    generate_scrubber_signature([scrubber])
+  end
+
+  defp generate_scrubber_signature(scrubbers) do
+    Enum.reduce(scrubbers, "", fn scrubber, signature ->
+        # If a scrubber does not have a version(e.g HtmlSanitizeEx.Scrubber) it is assumed it is always 0)
+        version = if Kernel.function_exported?(scrubber, :version, 0) do
+          scrubber.version
+        else
+          0
+        end
+        "#{signature}#{to_string(scrubber)}#{version}"
+    end)
+  end
 end

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 06d44451e4e55be7012cfcf28d3056422f74b68c..5e58215616872b1a5f754bbca42278f7e8b80f48 100644 (file)
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -128,8 +128,7 @@ defmodule Pleroma.Web.CommonAPI do
              |> Enum.reduce(%{}, fn {name, file}, acc ->
                Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
              end)
-           ),
-         object <- Map.put(object, "scrubber_cache", %{}) do
+           ) do
       res =
         ActivityPub.create(%{
           to: to,

diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 759bd62af8751aaae2bd32bbe01f027e35921ba3..813eb4093163fe8e8fb31f107987c79eb1e9ac82 100644 (file)
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.CommonAPI.Utils do
   alias Calendar.Strftime
   alias Comeonin.Pbkdf2
-  alias Pleroma.{Activity, Formatter, Object, Repo, HTML}
+  alias Pleroma.{Activity, Formatter, Object, Repo}
   alias Pleroma.User
   alias Pleroma.Web
   alias Pleroma.Web.ActivityPub.Utils
@@ -262,83 +262,4 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end)
   end
 
-  def get_scrubbed_html_for_object(content, scrubber, activity) when is_atom(scrubber) do
-    get_scrubbed_html_for_object(content, [scrubber], activity)
-  end
-  @doc """
-  Get sanitized HTML from cache, or scrub it and save to cache.
-  """
-  def get_scrubbed_html_for_object(
-        content,
-        scrubbers,
-        %{data: %{"object" => object}} = activity
-      ) do
-    scrubber_cache =
-      if is_list(object["scrubber_cache"]) do
-        object["scrubber_cache"]
-      else
-        []
-      end
-
-    signature = generate_scrubber_signature(scrubbers)
-
-    {new_scrubber_cache, scrubbed_html} =
-      Enum.map_reduce(scrubber_cache, nil, fn
-        entry, content ->
-          if Map.keys(entry["scrubbers"]) == Map.keys(signature) do
-            if entry["scrubbers"] == signature do
-              {entry, entry["content"]}
-            else
-              # Remove the entry if scrubber version is outdated
-              {nil, nil}
-            end
-          else
-            {entry, content}
-          end
-      end)
-
-    # Remove nil objects
-    new_scrubber_cache = Enum.reject(new_scrubber_cache, &is_nil/1)
-
-    if scrubbed_html == nil or new_scrubber_cache != scrubber_cache do
-      scrubbed_html = HTML.filter_tags(content, scrubbers)
-
-      new_scrubber_cache = [
-        %{:scrubbers => signature, :content => scrubbed_html} | new_scrubber_cache
-      ]
-
-      update_scrubber_cache(activity, new_scrubber_cache)
-      scrubbed_html
-    else
-      scrubbed_html
-    end
-  end
-
-  defp generate_scrubber_signature(scrubbers) do
-    Enum.reduce(scrubbers, %{}, fn scrubber, signature ->
-      Map.put(
-        signature,
-        to_string(scrubber),
-        # If a scrubber does not have a version(e.g HtmlSanitizeEx.Scrubber) it is assumed it is always 0)
-        if Kernel.function_exported?(scrubber, :version, 0) do
-          scrubber.version
-        else
-          0
-        end
-      )
-    end)
-  end
-
-  defp update_scrubber_cache(activity, scrubber_cache) do
-    cng =
-      Object.change(activity, %{
-        data: Kernel.put_in(activity.data, ["object", "scrubber_cache"], scrubber_cache)
-      })
-
-    {:ok, _struct} = Repo.update(cng)
-  end
-
-  def get_stripped_html_for_object(content, activity) do
-    get_scrubbed_html_for_object(content, [HtmlSanitizeEx.Scrubber.StripTags], activity)
-  end
 end

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 05ed602d595728f48af862152c8bb07cd65a4b49..8a57a233a71a509642ca5a87c1431789f3da997e 100644 (file)
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.HTML
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Object
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.MastodonAPI.AccountView
@@ -120,7 +121,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     content =
       object
       |> render_content()
-      |> Utils.get_scrubbed_html_for_object(User.html_filter_policy(opts[:for]), activity)
+      |> Object.get_cached_scrubbed_html(User.html_filter_policy(opts[:for]), activity)
 
     %{
       id: to_string(activity.id),

diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 7d0dea8c256c97dfaa3e3d66d1ead9ebc4c89eb4..4c29e03ce02e97de7b7e0c11a93d7846bb7e5471 100644 (file)
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -244,14 +244,14 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
     html =
       content
-      |> Utils.get_scrubbed_html_for_object(User.html_filter_policy(opts[:for]), activity)
+      |> Object.get_cached_scrubbed_html(User.html_filter_policy(opts[:for]), activity)
       |> Formatter.emojify(object["emoji"])
 
     text =
       if content do
         content
         |> String.replace(~r/<br\s?\/?>/, "\n")
-        |> Utils.get_stripped_html_for_object(activity)
+        |> Object.get_cached_stripped_html(activity)
       end
 
     reply_parent = Activity.get_in_reply_to_activity(activity)