Merge branch 'feat/rich-media-head' into 'develop'

RichMedia: Do a HEAD request to check content type/length

See merge request pleroma/pleroma!2995

diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 752ca9f8137c6cf6321cc96e0a7bef49e23b7a49..b7852c6e3fec025ec84bbe42511b4085ed11a6f7 100644 (file)
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -96,6 +96,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do
         @rich_media_options
       end
 
-    Pleroma.HTTP.get(url, headers, adapter: options)
+    head_check =
+      case Pleroma.HTTP.head(url, headers, adapter: options) do
+        # If the HEAD request didn't reach the server for whatever reason,
+        # we assume the GET that comes right after won't either
+        {:error, _} = e ->
+          e
+
+        {:ok, %Tesla.Env{status: 200, headers: headers}} ->
+          with :ok <- check_content_type(headers),
+               :ok <- check_content_length(headers),
+               do: :ok
+
+        _ ->
+          :ok
+      end
+
+    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, adapter: options)
+  end
+
+  defp check_content_type(headers) do
+    case List.keyfind(headers, "content-type", 0) do
+      {_, content_type} ->
+        case Plug.Conn.Utils.media_type(content_type) do
+          {:ok, "text", "html", _} -> :ok
+          _ -> {:error, {:content_type, content_type}}
+        end
+
+      _ ->
+        :ok
+    end
+  end
+
+  @max_body @rich_media_options[:max_body]
+  defp check_content_length(headers) do
+    case List.keyfind(headers, "content-length", 0) do
+      {_, maybe_content_length} ->
+        case Integer.parse(maybe_content_length) do
+          {content_length, ""} when content_length <= @max_body -> :ok
+          {_, ""} -> {:error, :body_too_large}
+          _ -> :ok
+        end
+
+      _ ->
+        :ok
+    end
   end
 end

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index e98c743caa7d4be1214b4a1157fc365d2305f571..49ba22c90911dcb62714e3e2d15e73e9321f2c83 100644 (file)
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -31,6 +31,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
         {:ok, _data} = res ->
           res
 
+        {:error, :body_too_large} = e ->
+          e
+
+        {:error, {:content_type, _}} ->
+          e
+
+        # The TTL is not set for the errors above, since they are unlikely to change
+        # with time
         {:error, _} = e ->
           ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
           Cachex.expire(:rich_media_cache, url, ttl)

diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
index a0ebf65d978febfdca68dce5c71729c04b5992ee..d9be248dc3c3bb54976c7b452f451e686067ef0e 100644 (file)
--- a/test/support/http_request_mock.ex
+++ b/test/support/http_request_mock.ex
@@ -1436,4 +1436,21 @@ defmodule HttpRequestMock do
        inspect(headers)
      }"}
   end
+
+  # Most of the rich media mocks are missing HEAD requests, so we just return 404.
+  @rich_media_mocks [
+    "https://example.com/ogp",
+    "https://example.com/ogp-missing-data",
+    "https://example.com/twitter-card"
+  ]
+  def head(url, _query, _body, _headers) when url in @rich_media_mocks do
+    {:ok, %Tesla.Env{status: 404, body: ""}}
+  end
+
+  def head(url, query, body, headers) do
+    {:error,
+     "Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{
+       inspect(headers)
+     }"}
+  end
 end

diff --git a/test/web/rich_media/parser_test.exs b/test/web/rich_media/parser_test.exs
index 1e09cbf842415776035baca2270c85e9df90a369..b8ef2cccf1beda51ab7cfe657cec79bf8380fe86 100644 (file)
--- a/test/web/rich_media/parser_test.exs
+++ b/test/web/rich_media/parser_test.exs
@@ -56,6 +56,27 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
 
       %{method: :get, url: "http://example.com/error"} ->
         {:error, :overload}
+
+      %{
+        method: :head,
+        url: "http://example.com/huge-page"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
+        }
+
+      %{
+        method: :head,
+        url: "http://example.com/pdf-file"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
+        }
+
+      %{method: :head} ->
+        %Tesla.Env{status: 404, body: "", headers: []}
     end)
 
     :ok
@@ -146,4 +167,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
   test "returns error if getting page was not successful" do
     assert {:error, :overload} = Parser.parse("http://example.com/error")
   end
+
+  test "does a HEAD request to check if the body is too large" do
+    assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page")
+  end
+
+  test "does a HEAD request to check if the body is html" do
+    assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file")
+  end
 end