Rich media: Add failure tracking

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 07bc6d77c4446866ab11bd5846e2b3d35a8178a8..4424c060f3d4ccf3ad4a5b45937cc9b901ab12aa 100644 (file)
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## unreleased-patch - ???
 
+### Added
+- Rich media failure tracking (along with `:failure_backoff` option)
+
 ### Fixed
 - Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers
 

diff --git a/config/config.exs b/config/config.exs
index ea8869d481b2f998b4256fa8a39b054259c63a68..ed37b93c0712429ea35f2cebf7472ca766a80391 100644 (file)
--- a/config/config.exs
+++ b/config/config.exs
@@ -412,6 +412,7 @@ config :pleroma, :rich_media,
     Pleroma.Web.RichMedia.Parsers.TwitterCard,
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ],
+  failure_backoff: 60_000,
   ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
 
 config :pleroma, :media_proxy,

diff --git a/config/description.exs b/config/description.exs
index 29a657333e50c564fbfeba4bf61be39d36dc19e0..5e08ba109d98b898d9a63c35e382a8d4ff68aca1 100644 (file)
--- a/config/description.exs
+++ b/config/description.exs
@@ -2385,6 +2385,13 @@ config :pleroma, :config_description, [
         suggestions: [
           Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
         ]
+      },
+      %{
+        key: :failure_backoff,
+        type: :integer,
+        description:
+          "Amount of milliseconds after request failure, during which the request will not be retried.",
+        suggestions: [60_000]
       }
     ]
   },

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 2f440adf4c6ab637543c67b71849743348d852e7..a9a650fab84a160bca00ea297ed860eb215064ba 100644 (file)
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -361,6 +361,7 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
 * `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
 * `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"].
 * `parsers`: list of Rich Media parsers.
+* `failure_backoff`: Amount of milliseconds after request failure, during which the request will not be retried.
 
 ## HTTP server
 

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index e9aa2dd03d69e810caff67e85cfdbf879979991a..e98c743caa7d4be1214b4a1157fc365d2305f571 100644 (file)
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -17,14 +17,25 @@ defmodule Pleroma.Web.RichMedia.Parser do
   else
     @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url) do
-      Cachex.fetch!(:rich_media_cache, url, fn _ ->
-        with {:ok, data} <- parse_url(url) do
-          {:commit, {:ok, data}}
-        else
-          error -> {:ignore, error}
-        end
-      end)
-      |> set_ttl_based_on_image(url)
+      with {:ok, data} <- get_cached_or_parse(url),
+           {:ok, _} <- set_ttl_based_on_image(data, url) do
+        {:ok, data}
+      else
+        error ->
+          Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
+      end
+    end
+
+    defp get_cached_or_parse(url) do
+      case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
+        {:ok, _data} = res ->
+          res
+
+        {:error, _} = e ->
+          ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
+          Cachex.expire(:rich_media_cache, url, ttl)
+          e
+      end
     end
   end
 
@@ -50,24 +61,23 @@ defmodule Pleroma.Web.RichMedia.Parser do
       config :pleroma, :rich_media,
         ttl_setters: [MyModule]
   """
-  @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
-          {:ok, map()} | {:error, any()}
-  def set_ttl_based_on_image({:ok, data}, url) do
-    with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
-         {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
-      Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
-      {:ok, data}
-    else
+  @spec set_ttl_based_on_image(map(), String.t()) ::
+          {:ok, Integer.t() | :noop} | {:error, :no_key}
+  def set_ttl_based_on_image(data, url) do
+    case get_ttl_from_image(data, url) do
+      {:ok, ttl} when is_number(ttl) ->
+        ttl = ttl * 1000
+
+        case Cachex.expire_at(:rich_media_cache, url, ttl) do
+          {:ok, true} -> {:ok, ttl}
+          {:ok, false} -> {:error, :no_key}
+        end
+
       _ ->
-        {:ok, data}
+        {:ok, :noop}
     end
   end
 
-  def set_ttl_based_on_image({:error, _} = error, _) do
-    Logger.error("parsing error: #{inspect(error)}")
-    error
-  end
-
   defp get_ttl_from_image(data, url) do
     [:rich_media, :ttl_setters]
     |> Pleroma.Config.get()

diff --git a/test/web/rich_media/aws_signed_url_test.exs b/test/web/rich_media/aws_signed_url_test.exs
index a21f3c935ab86eb3a8c38ecb9e7572f768830421..1ceae1a31569adc673fec9b2308dc54861787d57 100644 (file)
--- a/test/web/rich_media/aws_signed_url_test.exs
+++ b/test/web/rich_media/aws_signed_url_test.exs
@@ -55,7 +55,7 @@ defmodule Pleroma.Web.RichMedia.TTL.AwsSignedUrlTest do
 
     Cachex.put(:rich_media_cache, url, metadata)
 
-    Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url)
+    Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image(metadata, url)
 
     {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)