Merge remote-tracking branch 'remotes/origin/develop' into 2168-media-preview-proxy

# Conflicts:
#	config/config.exs
#	lib/pleroma/web/media_proxy/media_proxy.ex
#	mix.lock
#	test/web/media_proxy/media_proxy_test.exs

diff --cc .gitlab-ci.yml
Simple merge

diff --cc config/config.exs
index c8b6c7fad0ffe581ccaed93cb770f99e9bc59904,2d3f35e70253cc3cf654ad4e792a8fecbb0f07c8..9d8d43c54107399060905e8e9f5ad24a4d7cb7b3
--- 1/config/config.exs
--- 2/config/config.exs
+++ b/config/config.exs
@@@ -717,10 -704,8 +716,12 @@@ config :tzdata, :http_client, Pleroma.H
  
  config :ex_aws, http_client: Pleroma.HTTP.ExAws
  
+ config :pleroma, :instances_favicons, enabled: false
+ 
 +config :pleroma, :exexec,
 +  root_mode: false,
 +  options: %{}
 +
  # Import environment specific config. This must remain at the bottom
  # of this file so it overrides the configuration defined above.
  import_config "#{Mix.env()}.exs"

diff --cc lib/pleroma/reverse_proxy/reverse_proxy.ex
Simple merge

diff --cc lib/pleroma/web/mastodon_api/views/status_view.ex
Simple merge

diff --cc lib/pleroma/web/media_proxy/media_proxy.ex
index 1b6242cb4d0fa9fad178216094956d9f088145a9,dfbfcea6bc74620504e9a0b339d8ca315b43e066..217c3f92263c65fa76be6b51e1536f6ed081aa3c
--- 1/lib/pleroma/web/media_proxy/media_proxy.ex
--- 2/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@@ -53,44 -53,36 +53,50 @@@ defmodule Pleroma.Web.MediaProxy d
      end
    end
  
 -  defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
 +  # Note: routing all URLs to preview handler (even local and whitelisted).
 +  #   Preview handler will call url/1 on decoded URLs, and applicable ones will detour media proxy.
 +  def preview_url(url) do
 +    if preview_enabled?() do
 +      encode_preview_url(url)
 +    else
 +      url
 +    end
 +  end
  
 -  defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
 +  def enabled?, do: Config.get([:media_proxy, :enabled], false)
  
 -  defp whitelisted?(url) do
 +  # Note: media proxy must be enabled for media preview proxy in order to load all
 +  #   non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
 +  def preview_enabled?, do: enabled?() and Config.get([:media_preview_proxy, :enabled], false)
 +
 +  def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
 +
 +  def whitelisted?(url) do
      %{host: domain} = URI.parse(url)
  
-     mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])
- 
-     upload_base_url_domain =
-       if !is_nil(Config.get([Upload, :base_url])) do
-         [URI.parse(Config.get([Upload, :base_url])).host]
+     mediaproxy_whitelist_domains =
+       [:media_proxy, :whitelist]
+       |> Config.get()
+       |> Enum.map(&maybe_get_domain_from_url/1)
+ 
+     whitelist_domains =
+       if base_url = Config.get([Upload, :base_url]) do
+         %{host: base_domain} = URI.parse(base_url)
+         [base_domain | mediaproxy_whitelist_domains]
        else
-         []
+         mediaproxy_whitelist_domains
        end
  
-     whitelist = mediaproxy_whitelist ++ upload_base_url_domain
+     domain in whitelist_domains
+   end
  
-     Enum.any?(whitelist, fn pattern ->
-       String.equivalent?(domain, pattern)
-     end)
+   defp maybe_get_domain_from_url("http" <> _ = url) do
+     URI.parse(url).host
    end
  
 -  def encode_url(url) do
+   defp maybe_get_domain_from_url(domain), do: domain
+ 
 +  defp base64_sig64(url) do
      base64 = Base.url_encode64(url, @base64_opts)
  
      sig64 =
@@@ -130,10 -110,10 +136,10 @@@
      if path = URI.parse(url_or_path).path, do: Path.basename(path)
    end
  
 -  def build_url(sig_base64, url_base64, filename \\ nil) do
 +  defp proxy_url(path, sig_base64, url_base64, filename) do
      [
-       Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()),
+       Config.get([:media_proxy, :base_url], Web.base_url()),
 -      "proxy",
 +      path,
        sig_base64,
        url_base64,
        filename

diff --cc lib/pleroma/web/router.ex
Simple merge

diff --cc mix.exs
Simple merge

diff --cc mix.lock
index a1d0bf0d294e11182dec6ed900dcae5cc841b193,8dd37a40fc5deb77a483ba0d98b5d2def8929e92..d26afef7ad975551d580cbfc955ba86b214a5372
--- 1/mix.lock
--- 2/mix.lock
+++ b/mix.lock
@@@ -27,12 -28,10 +28,12 @@@
    "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"},
    "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
    "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"},
-   "ecto": {:hex, :ecto, "3.4.4", "a2c881e80dc756d648197ae0d936216c0308370332c5e77a2325a10293eef845", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4bd3ad62abc3b21fb629f0f7a3dab23a192fca837d257dd08449fba7373561"},
+   "ecto": {:hex, :ecto, "3.4.5", "2bcd262f57b2c888b0bd7f7a28c8a48aa11dc1a2c6a858e45dd8f8426d504265", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8c6d1d4d524559e9b7a062f0498e2c206122552d63eacff0a6567ffe7a8e8691"},
    "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
-   "ecto_sql": {:hex, :ecto_sql, "3.3.4", "aa18af12eb875fbcda2f75e608b3bd534ebf020fc4f6448e4672fcdcbb081244", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4 or ~> 3.3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5eccbdbf92e3c6f213007a82d5dbba4cd9bb659d1a21331f89f408e4c0efd7a8"},
+   "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"},
 +  "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"},
    "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"},
 +  "erlexec": {:hex, :erlexec, "1.10.9", "3cbb3476f942bfb8b68b85721c21c1835061cf6dd35f5285c2362e85b100ddc7", [:rebar3], [], "hexpm", "271e5b5f2d91cdb9887efe74d89026c199bfc69f074cade0d08dab60993fa14e"},
    "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"},
    "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"},
    "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
@@@ -78,9 -76,8 +79,9 @@@
    "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
    "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"},
    "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
-   "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"},
+   "oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"},
    "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},
 +  "p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"},
    "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
    "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"},
    "phoenix": {:hex, :phoenix, "1.4.13", "67271ad69b51f3719354604f4a3f968f83aa61c19199343656c9caee057ff3b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab765a0feddb81fc62e2116c827b5f068df85159c162bee760745276ad7ddc1b"},

diff --cc test/web/media_proxy/media_proxy_test.exs
index ac5d8fd32a60f8f82e4be9c28487eb4d73962f17,72885cfdd09f0bb081f2e61a9625d830f48e6737..06990464ff2dc1cac0bcb4c9ca36f37f48da7efe
--- 1/test/web/media_proxy/media_proxy_test.exs
--- 2/test/web/media_proxy/media_proxy_test.exs
+++ b/test/web/media_proxy/media_proxy_test.exs
@@@ -85,69 -75,9 +82,71 @@@ defmodule Pleroma.Web.MediaProxyTest d
        assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature}
      end
  
 +    def test_verify_request_path_and_url(request_path, url, expected_result) do
 +      assert MediaProxy.verify_request_path_and_url(request_path, url) == expected_result
 +
 +      assert MediaProxy.verify_request_path_and_url(
 +               %Plug.Conn{
 +                 params: %{"filename" => Path.basename(request_path)},
 +                 request_path: request_path
 +               },
 +               url
 +             ) == expected_result
 +    end
 +
 +    test "if first arg of `verify_request_path_and_url/2` is a Plug.Conn without \"filename\" " <>
 +           "parameter, `verify_request_path_and_url/2` returns :ok " do
 +      assert MediaProxy.verify_request_path_and_url(
 +               %Plug.Conn{params: %{}, request_path: "/some/path"},
 +               "https://instance.com/file.jpg"
 +             ) == :ok
 +
 +      assert MediaProxy.verify_request_path_and_url(
 +               %Plug.Conn{params: %{}, request_path: "/path/to/file.jpg"},
 +               "https://instance.com/file.jpg"
 +             ) == :ok
 +    end
 +
 +    test "`verify_request_path_and_url/2` preserves the encoded or decoded path" do
 +      test_verify_request_path_and_url(
 +        "/Hello world.jpg",
 +        "http://pleroma.social/Hello world.jpg",
 +        :ok
 +      )
 +
 +      test_verify_request_path_and_url(
 +        "/Hello%20world.jpg",
 +        "http://pleroma.social/Hello%20world.jpg",
 +        :ok
 +      )
 +
 +      test_verify_request_path_and_url(
 +        "/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
 +        "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
 +        :ok
 +      )
 +
 +      test_verify_request_path_and_url(
 +        "/my%2Flong%2Furl%2F2019%2F07%2FS",
 +        "http://pleroma.social/my%2Flong%2Furl%2F2019%2F07%2FS.jpg",
 +        {:wrong_filename, "my%2Flong%2Furl%2F2019%2F07%2FS.jpg"}
 +      )
 +    end
 +
 +    test "encoded url are tried to match for proxy as `conn.request_path` encodes the url" do
 +      # conn.request_path will return encoded url
 +      request_path = "/ANALYSE-DAI-_-LE-STABLECOIN-100-D%C3%89CENTRALIS%C3%89-BQ.jpg"
 +
 +      assert MediaProxy.verify_request_path_and_url(
 +               request_path,
 +               "https://mydomain.com/uploads/2019/07/ANALYSE-DAI-_-LE-STABLECOIN-100-DÉCENTRALISÉ-BQ.jpg"
 +             ) == :ok
++      assert MediaProxy.decode_url(sig, base64) == {:error, :invalid_signature}
 +    end
 +
      test "uses the configured base_url" do
-       clear_config([:media_proxy, :base_url], "https://cache.pleroma.social")
+       base_url = "https://cache.pleroma.social"
+       clear_config([:media_proxy, :base_url], base_url)
  
        url = "https://pleroma.soykaf.com/static/logo.png"
        encoded = MediaProxy.url(url)