1 # Pleroma: A lightweight social networking server
2 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
3 # SPDX-License-Identifier: AGPL-3.0-only
5 defmodule Pleroma.ReverseProxy do
6 @range_headers ~w(range if-range)
7 @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
8 ~w(if-unmodified-since if-none-match) ++ @range_headers
9 @resp_cache_headers ~w(etag date last-modified)
10 @keep_resp_headers @resp_cache_headers ++
11 ~w(content-length content-type content-disposition content-encoding) ++
12 ~w(content-range accept-ranges vary)
13 @default_cache_control_header "public, max-age=1209600"
14 @valid_resp_codes [200, 206, 304]
15 @max_read_duration :timer.seconds(30)
16 @max_body_length :infinity
17 @failed_request_ttl :timer.seconds(60)
20 def max_read_duration_default, do: @max_read_duration
21 def default_cache_control_header, do: @default_cache_control_header
26 Pleroma.ReverseProxy.call(conn, url, options)
28 It is not meant to be added into a plug pipeline, but to be called from another plug or controller.
30 Supports `#{inspect(@methods)}` HTTP methods, and only allows `#{inspect(@valid_resp_codes)}` status codes.
32 Responses are chunked to the client while downloading from the upstream.
34 Some request / responses headers are preserved:
36 * request: `#{inspect(@keep_req_headers)}`
37 * response: `#{inspect(@keep_resp_headers)}`
41 * `redirect_on_failure` (default `false`). Redirects the client to the real remote URL if there's any HTTP
42 errors. Any error during body processing will not be redirected as the response is chunked. This may expose
43 remote URL, clients IPs, ….
45 * `max_body_length` (default `#{inspect(@max_body_length)}`): limits the content length to be approximately the
46 specified length. It is validated with the `content-length` header and also verified when proxying.
48 * `max_read_duration` (default `#{inspect(@max_read_duration)}` ms): the total time the connection is allowed to
49 read from the remote upstream.
51 * `failed_request_ttl` (default `#{inspect(@failed_request_ttl)}` ms): the time the failed request is cached and cannot be retried.
53 * `inline_content_types`:
54 * `true` will not alter `content-disposition` (up to the upstream),
55 * `false` will add `content-disposition: attachment` to any request,
56 * a list of whitelisted content types
58 * `keep_user_agent` will forward the client's user-agent to the upstream. This may be useful if the upstream is
59 doing content transformation (encoding, …) depending on the request.
61 * `req_headers`, `resp_headers` additional headers.
63 * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
66 @default_options [pool: :media]
68 @inline_content_types [
85 {:keep_user_agent, boolean}
86 | {:max_read_duration, :timer.time() | :infinity}
87 | {:max_body_length, non_neg_integer() | :infinity}
88 | {:failed_request_ttl, :timer.time() | :infinity}
90 | {:req_headers, [{String.t(), String.t()}]}
91 | {:resp_headers, [{String.t(), String.t()}]}
92 | {:inline_content_types, boolean() | [String.t()]}
93 | {:redirect_on_failure, boolean()}
95 @spec call(Plug.Conn.t(), url :: String.t(), [option()]) :: Plug.Conn.t()
96 def call(_conn, _url, _opts \\ [])
98 def call(conn = %{method: method}, url, opts) when method in @methods do
99 client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
101 req_headers = build_req_headers(conn.req_headers, opts)
104 if filename = Pleroma.Web.MediaProxy.filename(url) do
105 Keyword.put_new(opts, :attachment_name, filename)
110 with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
111 {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
113 header_length_constraint(
115 Keyword.get(opts, :max_body_length, @max_body_length)
117 response(conn, client, url, code, headers, opts)
121 |> error_or_redirect(url, 500, "Request failed", opts)
124 {:ok, code, headers} ->
125 head_response(conn, url, code, headers, opts)
128 {:error, {:invalid_http_response, code}} ->
129 Logger.error("#{__MODULE__}: request to #{inspect(url)} failed with HTTP status #{code}")
130 track_failed_url(url, code, opts)
133 |> error_or_redirect(
136 "Request failed: " <> Plug.Conn.Status.reason_phrase(code),
142 Logger.error("#{__MODULE__}: request to #{inspect(url)} failed: #{inspect(error)}")
143 track_failed_url(url, error, opts)
146 |> error_or_redirect(url, 500, "Request failed", opts)
151 def call(conn, _, _) do
153 |> send_resp(400, Plug.Conn.Status.reason_phrase(400))
157 defp request(method, url, headers, opts) do
158 Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
159 method = method |> String.downcase() |> String.to_existing_atom()
161 case client().request(method, url, headers, "", opts) do
162 {:ok, code, headers, client} when code in @valid_resp_codes ->
163 {:ok, code, downcase_headers(headers), client}
165 {:ok, code, headers} when code in @valid_resp_codes ->
166 {:ok, code, downcase_headers(headers)}
169 {:error, {:invalid_http_response, code}}
172 {:error, {:invalid_http_response, code}}
179 defp response(conn, client, url, status, headers, opts) do
180 Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}")
184 |> put_resp_headers(build_resp_headers(headers, opts))
185 |> send_chunked(status)
186 |> chunk_reply(client, opts)
192 {:error, :closed, conn} ->
193 client().close(client)
196 {:error, error, conn} ->
198 "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}"
201 client().close(client)
206 defp chunk_reply(conn, client, opts) do
207 chunk_reply(conn, client, opts, 0, 0)
210 defp chunk_reply(conn, client, opts, sent_so_far, duration) do
211 with {:ok, duration} <-
214 Keyword.get(opts, :max_read_duration, @max_read_duration)
216 {:ok, data, client} <- client().stream_body(client),
217 {:ok, duration} <- increase_read_duration(duration),
218 sent_so_far = sent_so_far + byte_size(data),
220 body_size_constraint(
222 Keyword.get(opts, :max_body_length, @max_body_length)
224 {:ok, conn} <- chunk(conn, data) do
225 chunk_reply(conn, client, opts, sent_so_far, duration)
228 {:error, error} -> {:error, error, conn}
232 defp head_response(conn, url, code, headers, opts) do
233 Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}")
236 |> put_resp_headers(build_resp_headers(headers, opts))
237 |> send_resp(code, "")
240 defp error_or_redirect(conn, url, code, body, opts) do
241 if Keyword.get(opts, :redirect_on_failure, false) do
243 |> Phoenix.Controller.redirect(external: url)
247 |> send_resp(code, body)
252 defp downcase_headers(headers) do
253 Enum.map(headers, fn {k, v} ->
254 {String.downcase(k), v}
258 defp get_content_type(headers) do
260 List.keyfind(headers, "content-type", 0, {"content-type", "application/octet-stream"})
262 [content_type | _] = String.split(content_type, ";")
266 defp put_resp_headers(conn, headers) do
267 Enum.reduce(headers, conn, fn {k, v}, conn ->
268 put_resp_header(conn, k, v)
272 defp build_req_headers(headers, opts) do
274 |> downcase_headers()
275 |> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
276 |> build_req_range_or_encoding_header(opts)
277 |> build_req_user_agent_header(opts)
278 |> Keyword.merge(Keyword.get(opts, :req_headers, []))
281 # Disable content-encoding if any @range_headers are requested (see #1823).
282 defp build_req_range_or_encoding_header(headers, _opts) do
283 range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end)
285 if range? && List.keymember?(headers, "accept-encoding", 0) do
286 List.keydelete(headers, "accept-encoding", 0)
292 defp build_req_user_agent_header(headers, opts) do
293 if Keyword.get(opts, :keep_user_agent, false) do
298 {"user-agent", Pleroma.Application.user_agent()}
305 defp build_resp_headers(headers, opts) do
307 |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
308 |> build_resp_cache_headers(opts)
309 |> build_resp_content_disposition_header(opts)
310 |> Keyword.merge(Keyword.get(opts, :resp_headers, []))
313 defp build_resp_cache_headers(headers, _opts) do
314 has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
318 # There's caching header present but no cache-control -- we need to set our own
319 # as Plug defaults to "max-age=0, private, must-revalidate"
324 {"cache-control", @default_cache_control_header}
332 {"cache-control", @default_cache_control_header}
337 defp build_resp_content_disposition_header(headers, opts) do
338 opt = Keyword.get(opts, :inline_content_types, @inline_content_types)
340 content_type = get_content_type(headers)
344 is_list(opt) && !Enum.member?(opt, content_type) -> true
352 {{"content-disposition", content_disposition_string}, _} =
353 List.keytake(headers, "content-disposition", 0)
357 ~r/filename="((?:[^"\\]|\\.)*)"/u,
358 content_disposition_string || "",
359 capture: :all_but_first
364 MatchError -> Keyword.get(opts, :attachment_name, "attachment")
367 disposition = "attachment; filename=\"#{name}\""
369 List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
375 defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do
376 with {_, size} <- List.keyfind(headers, "content-length", 0),
377 {size, _} <- Integer.parse(size),
378 true <- size <= limit do
382 {:error, :body_too_large}
389 defp header_length_constraint(_, _), do: :ok
391 defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
392 {:error, :body_too_large}
395 defp body_size_constraint(_, _), do: :ok
397 defp check_read_duration(nil = _duration, max), do: check_read_duration(@max_read_duration, max)
399 defp check_read_duration(duration, max)
400 when is_integer(duration) and is_integer(max) and max > 0 do
402 {:error, :read_duration_exceeded}
404 {:ok, {duration, :erlang.system_time(:millisecond)}}
408 defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
410 defp increase_read_duration({previous_duration, started})
411 when is_integer(previous_duration) and is_integer(started) do
412 duration = :erlang.system_time(:millisecond) - started
413 {:ok, previous_duration + duration}
416 defp increase_read_duration(_) do
417 {:ok, :no_duration_limit, :no_duration_limit}
420 defp client, do: Pleroma.ReverseProxy.Client
422 defp track_failed_url(url, error, opts) do
424 unless error in [:body_too_large, 400, 204] do
425 Keyword.get(opts, :failed_request_ttl, @failed_request_ttl)
430 Cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)