1 # Pleroma: A lightweight social networking server
2 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
3 # SPDX-License-Identifier: AGPL-3.0-only
5 defmodule Pleroma.ReverseProxy do
6 @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
7 ~w(if-unmodified-since if-none-match if-range range)
8 @resp_cache_headers ~w(etag date last-modified cache-control)
9 @keep_resp_headers @resp_cache_headers ++
10 ~w(content-type content-disposition content-encoding content-range) ++
11 ~w(accept-ranges vary)
12 @default_cache_control_header "public, max-age=1209600"
13 @valid_resp_codes [200, 206, 304]
14 @max_read_duration :timer.seconds(30)
15 @max_body_length :infinity
16 @failed_request_ttl :timer.seconds(60)
22 Pleroma.ReverseProxy.call(conn, url, options)
24 It is not meant to be added into a plug pipeline, but to be called from another plug or controller.
26 Supports `#{inspect(@methods)}` HTTP methods, and only allows `#{inspect(@valid_resp_codes)}` status codes.
28 Responses are chunked to the client while downloading from the upstream.
30 Some request / responses headers are preserved:
32 * request: `#{inspect(@keep_req_headers)}`
33 * response: `#{inspect(@keep_resp_headers)}`
35 If no caching headers (`#{inspect(@resp_cache_headers)}`) are returned by upstream, `cache-control` will be
36 set to `#{inspect(@default_cache_control_header)}`.
40 * `redirect_on_failure` (default `false`). Redirects the client to the real remote URL if there's any HTTP
41 errors. Any error during body processing will not be redirected as the response is chunked. This may expose
42 remote URL, clients IPs, ….
44 * `max_body_length` (default `#{inspect(@max_body_length)}`): limits the content length to be approximately the
45 specified length. It is validated with the `content-length` header and also verified when proxying.
47 * `max_read_duration` (default `#{inspect(@max_read_duration)}` ms): the total time the connection is allowed to
48 read from the remote upstream.
50 * `failed_request_ttl` (default `#{inspect(@failed_request_ttl)}` ms): the time the failed request is cached and cannot be retried.
52 * `inline_content_types`:
53 * `true` will not alter `content-disposition` (up to the upstream),
54 * `false` will add `content-disposition: attachment` to any request,
55 * a list of whitelisted content types
57 * `keep_user_agent` will forward the client's user-agent to the upstream. This may be useful if the upstream is
58 doing content transformation (encoding, …) depending on the request.
60 * `req_headers`, `resp_headers` additional headers.
62 * `http`: options for [gun](https://github.com/ninenines/gun).
65 @default_options [pool: :media]
67 @inline_content_types [
84 {:keep_user_agent, boolean}
85 | {:max_read_duration, :timer.time() | :infinity}
86 | {:max_body_length, non_neg_integer() | :infinity}
87 | {:failed_request_ttl, :timer.time() | :infinity}
89 | {:req_headers, [{String.t(), String.t()}]}
90 | {:resp_headers, [{String.t(), String.t()}]}
91 | {:inline_content_types, boolean() | [String.t()]}
92 | {:redirect_on_failure, boolean()}
94 @spec call(Plug.Conn.t(), url :: String.t(), [option()]) :: Plug.Conn.t()
95 def call(_conn, _url, _opts \\ [])
97 def call(conn = %{method: method}, url, opts) when method in @methods do
98 client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
100 req_headers = build_req_headers(conn.req_headers, opts)
103 if filename = Pleroma.Web.MediaProxy.filename(url) do
104 Keyword.put_new(opts, :attachment_name, filename)
109 with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
110 {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
112 header_length_constraint(
114 Keyword.get(opts, :max_body_length, @max_body_length)
116 response(conn, client, url, code, headers, opts)
120 |> error_or_redirect(url, 500, "Request failed", opts)
123 {:ok, code, headers} ->
124 head_response(conn, url, code, headers, opts)
127 {:error, {:invalid_http_response, code}} ->
128 Logger.error("#{__MODULE__}: request to #{inspect(url)} failed with HTTP status #{code}")
129 track_failed_url(url, code, opts)
132 |> error_or_redirect(
135 "Request failed: " <> Plug.Conn.Status.reason_phrase(code),
141 Logger.error("#{__MODULE__}: request to #{inspect(url)} failed: #{inspect(error)}")
142 track_failed_url(url, error, opts)
145 |> error_or_redirect(url, 500, "Request failed", opts)
150 def call(conn, _, _) do
152 |> send_resp(400, Plug.Conn.Status.reason_phrase(400))
156 defp request(method, url, headers, opts) do
157 Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
158 method = method |> String.downcase() |> String.to_existing_atom()
160 case client().request(method, url, headers, "", opts) do
161 {:ok, code, headers, client} when code in @valid_resp_codes ->
162 {:ok, code, downcase_headers(headers), client}
164 {:ok, code, headers} when code in @valid_resp_codes ->
165 {:ok, code, downcase_headers(headers)}
168 {:error, {:invalid_http_response, code}}
175 defp response(conn, client, url, status, headers, opts) do
178 |> put_resp_headers(build_resp_headers(headers, opts))
179 |> send_chunked(status)
180 |> chunk_reply(client, opts)
186 {:error, :closed, conn} ->
187 client().close(client)
190 {:error, error, conn} ->
192 "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}"
195 client().close(client)
200 defp chunk_reply(conn, client, opts) do
201 chunk_reply(conn, client, opts, 0, 0)
204 defp chunk_reply(conn, client, opts, sent_so_far, duration) do
205 with {:ok, duration} <-
208 Keyword.get(opts, :max_read_duration, @max_read_duration)
210 {:ok, data, client} <- client().stream_body(client),
211 {:ok, duration} <- increase_read_duration(duration),
212 sent_so_far = sent_so_far + byte_size(data),
214 body_size_constraint(
216 Keyword.get(opts, :max_body_length, @max_body_length)
218 {:ok, conn} <- chunk(conn, data) do
219 chunk_reply(conn, client, opts, sent_so_far, duration)
222 {:error, error} -> {:error, error, conn}
226 defp head_response(conn, _url, code, headers, opts) do
228 |> put_resp_headers(build_resp_headers(headers, opts))
229 |> send_resp(code, "")
232 defp error_or_redirect(conn, url, code, body, opts) do
233 if Keyword.get(opts, :redirect_on_failure, false) do
235 |> Phoenix.Controller.redirect(external: url)
239 |> send_resp(code, body)
244 defp downcase_headers(headers) do
245 Enum.map(headers, fn {k, v} ->
246 {String.downcase(k), v}
250 defp get_content_type(headers) do
252 List.keyfind(headers, "content-type", 0, {"content-type", "application/octet-stream"})
254 [content_type | _] = String.split(content_type, ";")
258 defp put_resp_headers(conn, headers) do
259 Enum.reduce(headers, conn, fn {k, v}, conn ->
260 put_resp_header(conn, k, v)
264 defp build_req_headers(headers, opts) do
266 |> downcase_headers()
267 |> Enum.filter(fn {k, _} -> k in @keep_req_headers end)
269 headers = headers ++ Keyword.get(opts, :req_headers, [])
271 if Keyword.get(opts, :keep_user_agent, false) do
276 {"user-agent", Pleroma.Application.user_agent()}
284 defp build_resp_headers(headers, opts) do
286 |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end)
287 |> build_resp_cache_headers(opts)
288 |> build_resp_content_disposition_header(opts)
289 |> (fn headers -> headers ++ Keyword.get(opts, :resp_headers, []) end).()
292 defp build_resp_cache_headers(headers, _opts) do
293 has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
294 has_cache_control? = List.keymember?(headers, "cache-control", 0)
297 has_cache? && has_cache_control? ->
301 # There's caching header present but no cache-control -- we need to explicitely override it
302 # to public as Plug defaults to "max-age=0, private, must-revalidate"
303 List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
310 {"cache-control", @default_cache_control_header}
315 defp build_resp_content_disposition_header(headers, opts) do
316 opt = Keyword.get(opts, :inline_content_types, @inline_content_types)
318 content_type = get_content_type(headers)
322 is_list(opt) && !Enum.member?(opt, content_type) -> true
330 {{"content-disposition", content_disposition_string}, _} =
331 List.keytake(headers, "content-disposition", 0)
335 ~r/filename="((?:[^"\\]|\\.)*)"/u,
336 content_disposition_string || "",
337 capture: :all_but_first
342 MatchError -> Keyword.get(opts, :attachment_name, "attachment")
345 disposition = "attachment; filename=\"#{name}\""
347 List.keystore(headers, "content-disposition", 0, {"content-disposition", disposition})
353 defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do
354 with {_, size} <- List.keyfind(headers, "content-length", 0),
355 {size, _} <- Integer.parse(size),
356 true <- size <= limit do
360 {:error, :body_too_large}
367 defp header_length_constraint(_, _), do: :ok
369 defp body_size_constraint(size, limit) when is_integer(limit) and limit > 0 and size >= limit do
370 {:error, :body_too_large}
373 defp body_size_constraint(_, _), do: :ok
375 defp check_read_duration(duration, max)
376 when is_integer(duration) and is_integer(max) and max > 0 do
378 {:error, :read_duration_exceeded}
380 {:ok, {duration, :erlang.system_time(:millisecond)}}
384 defp check_read_duration(_, _), do: {:ok, :no_duration_limit, :no_duration_limit}
386 defp increase_read_duration({previous_duration, started})
387 when is_integer(previous_duration) and is_integer(started) do
388 duration = :erlang.system_time(:millisecond) - started
389 {:ok, previous_duration + duration}
392 defp increase_read_duration(_) do
393 {:ok, :no_duration_limit, :no_duration_limit}
396 defp client, do: Pleroma.ReverseProxy.Client
398 defp track_failed_url(url, error, opts) do
400 unless error in [:body_too_large, 400, 204] do
401 Keyword.get(opts, :failed_request_ttl, @failed_request_ttl)
406 Cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)