],
whitelist: []
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
+ method: :purge,
+ headers: [],
+ options: []
+
+config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
+
config :pleroma, :chat, enabled: true
config :phoenix, :format_encoders, json: Jason
"The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.",
suggestions: ["https://example.com"]
},
+ %{
+ key: :invalidation,
+ type: :keyword,
+ descpiption: "",
+ suggestions: [
+ enabled: true,
+ provider: Pleroma.Web.MediaProxy.Invalidation.Script
+ ],
+ children: [
+ %{
+ key: :enabled,
+ type: :boolean,
+ description: "Enables invalidate media cache"
+ },
+ %{
+ key: :provider,
+ type: :module,
+ description: "Module which will be used to cache purge.",
+ suggestions: [
+ Pleroma.Web.MediaProxy.Invalidation.Script,
+ Pleroma.Web.MediaProxy.Invalidation.Http
+ ]
+ }
+ ]
+ },
%{
key: :proxy_opts,
type: :keyword,
}
]
},
+ %{
+ group: :pleroma,
+ key: Pleroma.Web.MediaProxy.Invalidation.Http,
+ type: :group,
+ description: "HTTP invalidate settings",
+ children: [
+ %{
+ key: :method,
+ type: :atom,
+ description: "HTTP method of request. Default: :purge"
+ },
+ %{
+ key: :headers,
+ type: {:list, :tuple},
+ description: "HTTP headers of request.",
+ suggestions: [{"x-refresh", 1}]
+ },
+ %{
+ key: :options,
+ type: :keyword,
+ description: "Request options.",
+ suggestions: [params: %{ts: "xxx"}]
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
+ key: Pleroma.Web.MediaProxy.Invalidation.Script,
+ type: :group,
+ description: "Script invalidate settings",
+ children: [
+ %{
+ key: :script_path,
+ type: :string,
+ description: "Path to shell script. Which will run purge cache.",
+ suggestions: ["./installation/nginx-cache-purge.sh.example"]
+ }
+ ]
+ },
%{
group: :pleroma,
key: :gopher,
- Response:
- On success: `204`, empty response
- On failure:
- - 400 Bad Request `"Invalid parameters"` when `status` is missing
\ No newline at end of file
+ - 400 Bad Request `"Invalid parameters"` when `status` is missing
+
+## `GET /api/pleroma/admin/media_proxy_caches`
+
+### Get a list of all banned MediaProxy URLs in Cachex
+
+- Authentication: required
+- Params:
+- *optional* `page`: **integer** page number
+- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+
+- Response:
+
+``` json
+{
+ "urls": [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+}
+
+```
+
+## `POST /api/pleroma/admin/media_proxy_caches/delete`
+
+### Remove a banned MediaProxy URL from Cachex
+
+- Authentication: required
+- Params:
+ - `urls` (array)
+
+- Response:
+
+``` json
+{
+ "urls": [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+}
+
+```
+
+## `POST /api/pleroma/admin/media_proxy_caches/purge`
+
+### Purge a MediaProxy URL
+
+- Authentication: required
+- Params:
+ - `urls` (array)
+ - `ban` (boolean)
+
+- Response:
+
+``` json
+{
+ "urls": [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+}
+
+```
#### Pleroma.Web.MediaProxy.Invalidation.Script
-This strategy allow perform external bash script to purge cache.
+This strategy allow perform external shell script to purge cache.
Urls of attachments pass to script as arguments.
* `script_path`: path to external script.
This strategy allow perform custom http request to purge cache.
* `method`: http method. default is `purge`
-* `headers`: http headers. default is empty
-* `options`: request options. default is empty
+* `headers`: http headers.
+* `options`: request options.
Example:
```elixir
## $3 - (optional) the number of parallel processes to run for grep.
get_cache_files() {
local max_parallel=${3-16}
- find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u
+ find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u
}
## Removes an item from the given cache zone.
}
-purge $1
+purge $@
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
- build_cachex("failed_proxy_url", limit: 2500)
+ build_cachex("failed_proxy_url", limit: 2500),
+ build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
]
end
import Pleroma.Web.Gettext
require Logger
+ alias Pleroma.Web.MediaProxy
+
@behaviour Plug
# no slashes
@path "media"
%{query_params: %{"name" => name}} = conn ->
name = String.replace(name, "\"", "\\\"")
- conn
- |> put_resp_header("content-disposition", "filename=\"#{name}\"")
+ put_resp_header(conn, "content-disposition", "filename=\"#{name}\"")
conn ->
conn
with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false),
- {:ok, get_method} <- uploader.get_file(file) do
+ {:ok, get_method} <- uploader.get_file(file),
+ false <- media_is_deleted(conn, get_method) do
get_media(conn, get_method, proxy_remote, opts)
else
_ ->
def call(conn, _opts), do: conn
+ defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do
+ MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path)
+ end
+
+ defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url)
+
+ defp media_is_deleted(_, _), do: false
+
defp get_media(conn, {:static_dir, directory}, _, opts) do
static_opts =
Map.get(opts, :static_plug_opts)
--- /dev/null
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
+ use Pleroma.Web, :controller
+
+ alias Pleroma.Plugs.OAuthScopesPlug
+ alias Pleroma.Web.ApiSpec.Admin, as: Spec
+ alias Pleroma.Web.MediaProxy
+
+ plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+ plug(
+ OAuthScopesPlug,
+ %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
+ )
+
+ plug(
+ OAuthScopesPlug,
+ %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
+ )
+
+ action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+ defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
+
+ def index(%{assigns: %{user: _}} = conn, params) do
+ cursor =
+ :deleted_urls_cache
+ |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
+ |> :qlc.cursor()
+
+ urls =
+ case params.page do
+ 1 ->
+ :qlc.next_answers(cursor, params.page_size)
+
+ _ ->
+ :qlc.next_answers(cursor, (params.page - 1) * params.page_size)
+ :qlc.next_answers(cursor, params.page_size)
+ end
+
+ :qlc.delete_cursor(cursor)
+
+ render(conn, "index.json", urls: urls)
+ end
+
+ def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
+ MediaProxy.remove_from_deleted_urls(urls)
+ render(conn, "index.json", urls: urls)
+ end
+
+ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
+ MediaProxy.Invalidation.purge(urls)
+
+ if ban do
+ MediaProxy.put_in_deleted_urls(urls)
+ end
+
+ render(conn, "index.json", urls: urls)
+ end
+end
--- /dev/null
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
+ use Pleroma.Web, :view
+
+ def render("index.json", %{urls: urls}) do
+ %{urls: urls}
+ end
+end
--- /dev/null
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
+ alias OpenApiSpex.Operation
+ alias OpenApiSpex.Schema
+ alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+ import Pleroma.Web.ApiSpec.Helpers
+
+ def open_api_operation(action) do
+ operation = String.to_existing_atom("#{action}_operation")
+ apply(__MODULE__, operation, [])
+ end
+
+ def index_operation do
+ %Operation{
+ tags: ["Admin", "MediaProxyCache"],
+ summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
+ operationId: "AdminAPI.MediaProxyCacheController.index",
+ security: [%{"oAuth" => ["read:media_proxy_caches"]}],
+ parameters: [
+ Operation.parameter(
+ :page,
+ :query,
+ %Schema{type: :integer, default: 1},
+ "Page"
+ ),
+ Operation.parameter(
+ :page_size,
+ :query,
+ %Schema{type: :integer, default: 50},
+ "Number of statuses to return"
+ )
+ ],
+ responses: %{
+ 200 => success_response()
+ }
+ }
+ end
+
+ def delete_operation do
+ %Operation{
+ tags: ["Admin", "MediaProxyCache"],
+ summary: "Remove a banned MediaProxy URL from Cachex",
+ operationId: "AdminAPI.MediaProxyCacheController.delete",
+ security: [%{"oAuth" => ["write:media_proxy_caches"]}],
+ requestBody:
+ request_body(
+ "Parameters",
+ %Schema{
+ type: :object,
+ required: [:urls],
+ properties: %{
+ urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
+ }
+ },
+ required: true
+ ),
+ responses: %{
+ 200 => success_response(),
+ 400 => Operation.response("Error", "application/json", ApiError)
+ }
+ }
+ end
+
+ def purge_operation do
+ %Operation{
+ tags: ["Admin", "MediaProxyCache"],
+ summary: "Purge and optionally ban a MediaProxy URL",
+ operationId: "AdminAPI.MediaProxyCacheController.purge",
+ security: [%{"oAuth" => ["write:media_proxy_caches"]}],
+ requestBody:
+ request_body(
+ "Parameters",
+ %Schema{
+ type: :object,
+ required: [:urls],
+ properties: %{
+ urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
+ ban: %Schema{type: :boolean, default: true}
+ }
+ },
+ required: true
+ ),
+ responses: %{
+ 200 => success_response(),
+ 400 => Operation.response("Error", "application/json", ApiError)
+ }
+ }
+ end
+
+ defp success_response do
+ Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
+ type: :object,
+ properties: %{
+ urls: %Schema{
+ type: :array,
+ items: %Schema{
+ type: :string,
+ format: :uri,
+ description: "MediaProxy URLs"
+ }
+ }
+ }
+ })
+ end
+end
defmodule Pleroma.Web.MediaProxy.Invalidation do
@moduledoc false
- @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
+ @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
alias Pleroma.Config
+ alias Pleroma.Web.MediaProxy
- @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
+ @spec enabled?() :: boolean()
+ def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled])
+
+ @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
def purge(urls) do
- [:media_proxy, :invalidation, :enabled]
- |> Config.get()
- |> do_purge(urls)
+ prepared_urls = prepare_urls(urls)
+
+ if enabled?() do
+ do_purge(prepared_urls)
+ else
+ {:ok, prepared_urls}
+ end
end
- defp do_purge(true, urls) do
+ defp do_purge(urls) do
provider = Config.get([:media_proxy, :invalidation, :provider])
- options = Config.get(provider)
- provider.purge(urls, options)
+ provider.purge(urls, Config.get(provider))
end
- defp do_purge(_, _), do: :ok
+ def prepare_urls(urls) do
+ urls
+ |> List.wrap()
+ |> Enum.map(&MediaProxy.url/1)
+ end
end
@impl Pleroma.Web.MediaProxy.Invalidation
def purge(urls, opts) do
- method = Map.get(opts, :method, :purge)
- headers = Map.get(opts, :headers, [])
- options = Map.get(opts, :options, [])
+ method = Keyword.get(opts, :method, :purge)
+ headers = Keyword.get(opts, :headers, [])
+ options = Keyword.get(opts, :options, [])
Logger.debug("Running cache purge: #{inspect(urls)}")
end
end)
- {:ok, "success"}
+ {:ok, urls}
end
defp do_purge(method, url, headers, options) do
require Logger
@impl Pleroma.Web.MediaProxy.Invalidation
- def purge(urls, %{script_path: script_path} = _options) do
+ def purge(urls, opts \\ []) do
args =
urls
|> List.wrap()
|> Enum.uniq()
|> Enum.join(" ")
- path = Path.expand(script_path)
-
- Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
-
- case do_purge(path, [args]) do
- {result, exit_status} when exit_status > 0 ->
- Logger.error("Error while cache purge: #{inspect(result)}")
- {:error, inspect(result)}
-
- _ ->
- {:ok, "success"}
- end
+ opts
+ |> Keyword.get(:script_path)
+ |> do_purge([args])
+ |> handle_result(urls)
end
- def purge(_, _), do: {:error, "not found script path"}
-
- defp do_purge(path, args) do
+ defp do_purge(script_path, args) when is_binary(script_path) do
+ path = Path.expand(script_path)
+ Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}")
System.cmd(path, args)
rescue
- error -> {inspect(error), 1}
+ error -> error
+ end
+
+ defp do_purge(_, _), do: {:error, "not found script path"}
+
+ defp handle_result({_result, 0}, urls), do: {:ok, urls}
+ defp handle_result({:error, error}, urls), do: handle_result(error, urls)
+
+ defp handle_result(error, _) do
+ Logger.error("Error while cache purge: #{inspect(error)}")
+ {:error, inspect(error)}
end
end
alias Pleroma.Config
alias Pleroma.Upload
alias Pleroma.Web
+ alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false]
+ @spec in_deleted_urls(String.t()) :: boolean()
+ def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1)
+
+ def remove_from_deleted_urls(urls) when is_list(urls) do
+ Cachex.execute!(:deleted_urls_cache, fn cache ->
+ Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
+ end)
+ end
+
+ def remove_from_deleted_urls(url) when is_binary(url) do
+ Cachex.del(:deleted_urls_cache, url(url))
+ end
+
+ def put_in_deleted_urls(urls) when is_list(urls) do
+ Cachex.execute!(:deleted_urls_cache, fn cache ->
+ Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
+ end)
+ end
+
+ def put_in_deleted_urls(url) when is_binary(url) do
+ Cachex.put(:deleted_urls_cache, url(url), true)
+ end
+
def url(url) when is_nil(url) or url == "", do: nil
def url("/" <> _ = url), do: url
def url(url) do
- if disabled?() or local?(url) or whitelisted?(url) do
+ if disabled?() or not is_url_proxiable?(url) do
url
else
encode_url(url)
end
end
+ @spec is_url_proxiable?(String.t()) :: boolean()
+ def is_url_proxiable?(url) do
+ if local?(url) or whitelisted?(url) do
+ false
+ else
+ true
+ end
+ end
+
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
with config <- Pleroma.Config.get([:media_proxy], []),
true <- Keyword.get(config, :enabled, false),
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
+ {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)},
:ok <- filename_matches(params, conn.request_path, url) do
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
else
- false ->
+ error when error in [false, {:in_deleted_urls, true}] ->
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
{:error, :invalid_signature} ->
post("/oauth_app", OAuthAppController, :create)
patch("/oauth_app/:id", OAuthAppController, :update)
delete("/oauth_app/:id", OAuthAppController, :delete)
+
+ get("/media_proxy_caches", MediaProxyCacheController, :index)
+ post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
+ post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
},
_job
) do
- hrefs =
- Enum.flat_map(attachments, fn attachment ->
- Enum.map(attachment["url"], & &1["href"])
- end)
+ attachments
+ |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
+ |> fetch_objects
+ |> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
+ |> filter_objects
+ |> do_clean
- names = Enum.map(attachments, & &1["name"])
+ {:ok, :success}
+ end
+
+ def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
+ defp do_clean({object_ids, attachment_urls}) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
prefix =
"/"
)
- # find all objects for copies of the attachments, name and actor doesn't matter here
- object_ids_and_hrefs =
- from(o in Object,
- where:
- fragment(
- "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
- o.data,
- o.data,
- ^hrefs
- )
- )
- # The query above can be time consumptive on large instances until we
- # refactor how uploads are stored
- |> Repo.all(timeout: :infinity)
- # we should delete 1 object for any given attachment, but don't delete
- # files if there are more than 1 object for it
- |> Enum.reduce(%{}, fn %{
- id: id,
- data: %{
- "url" => [%{"href" => href}],
- "actor" => obj_actor,
- "name" => name
- }
- },
- acc ->
- Map.update(acc, href, %{id: id, count: 1}, fn val ->
- case obj_actor == actor and name in names do
- true ->
- # set id of the actor's object that will be deleted
- %{val | id: id, count: val.count + 1}
-
- false ->
- # another actor's object, just increase count to not delete file
- %{val | count: val.count + 1}
- end
- end)
- end)
- |> Enum.map(fn {href, %{id: id, count: count}} ->
- # only delete files that have single instance
- with 1 <- count do
- href
- |> String.trim_leading("#{base_url}/#{prefix}")
- |> uploader.delete_file()
-
- {id, href}
- else
- _ -> {id, nil}
- end
- end)
+ Enum.each(attachment_urls, fn href ->
+ href
+ |> String.trim_leading("#{base_url}/#{prefix}")
+ |> uploader.delete_file()
+ end)
- object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
+ delete_objects(object_ids)
+ end
- from(o in Object, where: o.id in ^object_ids)
- |> Repo.delete_all()
+ defp delete_objects([_ | _] = object_ids) do
+ Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
+ end
- object_ids_and_hrefs
- |> Enum.filter(fn {_, href} -> not is_nil(href) end)
- |> Enum.map(&elem(&1, 1))
- |> Pleroma.Web.MediaProxy.Invalidation.purge()
+ defp delete_objects(_), do: :ok
- {:ok, :success}
+ # we should delete 1 object for any given attachment, but don't delete
+ # files if there are more than 1 object for it
+ defp filter_objects(objects) do
+ Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
+ with 1 <- count do
+ {ids ++ [id], hrefs ++ [href]}
+ else
+ _ -> {ids ++ [id], hrefs}
+ end
+ end)
end
- def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
+ defp prepare_objects(objects, actor, names) do
+ objects
+ |> Enum.reduce(%{}, fn %{
+ id: id,
+ data: %{
+ "url" => [%{"href" => href}],
+ "actor" => obj_actor,
+ "name" => name
+ }
+ },
+ acc ->
+ Map.update(acc, href, %{id: id, count: 1}, fn val ->
+ case obj_actor == actor and name in names do
+ true ->
+ # set id of the actor's object that will be deleted
+ %{val | id: id, count: val.count + 1}
+
+ false ->
+ # another actor's object, just increase count to not delete file
+ %{val | count: val.count + 1}
+ end
+ end)
+ end)
+ end
+
+ defp fetch_objects(hrefs) do
+ from(o in Object,
+ where:
+ fragment(
+ "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
+ o.data,
+ o.data,
+ ^hrefs
+ )
+ )
+ # The query above can be time consumptive on large instances until we
+ # refactor how uploads are stored
+ |> Repo.all(timeout: :infinity)
+ end
end
--- /dev/null
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
+ use Pleroma.Web.ConnCase
+
+ import Pleroma.Factory
+ import Mock
+
+ alias Pleroma.Web.MediaProxy
+
+ setup do: clear_config([:media_proxy])
+
+ setup do
+ on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+ end
+
+ setup do
+ admin = insert(:user, is_admin: true)
+ token = insert(:oauth_admin_token, user: admin)
+
+ conn =
+ build_conn()
+ |> assign(:user, admin)
+ |> assign(:token, token)
+
+ Config.put([:media_proxy, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script)
+
+ {:ok, %{admin: admin, token: token, conn: conn}}
+ end
+
+ describe "GET /api/pleroma/admin/media_proxy_caches" do
+ test "shows banned MediaProxy URLs", %{conn: conn} do
+ MediaProxy.put_in_deleted_urls([
+ "http://localhost:4001/media/a688346.jpg",
+ "http://localhost:4001/media/fb1f4d.jpg"
+ ])
+
+ MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg")
+ MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg")
+ MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg")
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == [
+ "http://localhost:4001/media/fb1f4d.jpg",
+ "http://localhost:4001/media/a688346.jpg"
+ ]
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == [
+ "http://localhost:4001/media/gb1f44.jpg",
+ "http://localhost:4001/media/tb13f47.jpg"
+ ]
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
+ end
+ end
+
+ describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
+ test "deleted MediaProxy URLs from banned", %{conn: conn} do
+ MediaProxy.put_in_deleted_urls([
+ "http://localhost:4001/media/a688346.jpg",
+ "http://localhost:4001/media/fb1f4d.jpg"
+ ])
+
+ response =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
+ urls: ["http://localhost:4001/media/a688346.jpg"]
+ })
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
+ refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg")
+ assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg")
+ end
+ end
+
+ describe "POST /api/pleroma/admin/media_proxy_caches/purge" do
+ test "perform invalidates cache of MediaProxy", %{conn: conn} do
+ urls = [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+
+ with_mocks [
+ {MediaProxy.Invalidation.Script, [],
+ [
+ purge: fn _, _ -> {"ok", 0} end
+ ]}
+ ] do
+ response =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == urls
+
+ refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg")
+ refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg")
+ end
+ end
+
+ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do
+ urls = [
+ "http://example.com/media/a688346.jpg",
+ "http://example.com/media/fb1f4d.jpg"
+ ]
+
+ with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
+ response =
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
+ urls: urls,
+ ban: true
+ })
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == urls
+
+ assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg")
+ assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg")
+ end
+ end
+ end
+end
--- /dev/null
+defmodule Pleroma.Web.MediaProxy.InvalidationTest do
+ use ExUnit.Case
+ use Pleroma.Tests.Helpers
+
+ alias Pleroma.Config
+ alias Pleroma.Web.MediaProxy.Invalidation
+
+ import ExUnit.CaptureLog
+ import Mock
+ import Tesla.Mock
+
+ setup do: clear_config([:media_proxy])
+
+ setup do
+ on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+ end
+
+ describe "Invalidation.Http" do
+ test "perform request to clear cache" do
+ Config.put([:media_proxy, :enabled], false)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
+
+ Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
+ image_url = "http://example.com/media/example.jpg"
+ Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+
+ mock(fn
+ %{
+ method: :purge,
+ url: "http://example.com/media/example.jpg",
+ headers: [{"x-refresh", 1}]
+ } ->
+ %Tesla.Env{status: 200}
+ end)
+
+ assert capture_log(fn ->
+ assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+ assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+ assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+ end) =~ "Running cache purge: [\"#{image_url}\"]"
+ end
+ end
+
+ describe "Invalidation.Script" do
+ test "run script to clear cache" do
+ Config.put([:media_proxy, :enabled], false)
+ Config.put([:media_proxy, :invalidation, :enabled], true)
+ Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
+ Config.put([Invalidation.Script], script_path: "purge-nginx")
+
+ image_url = "http://example.com/media/example.jpg"
+ Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url)
+
+ with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
+ assert capture_log(fn ->
+ assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+ assert Invalidation.purge([image_url]) == {:ok, [image_url]}
+ assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url)
+ end) =~ "Running cache purge: [\"#{image_url}\"]"
+ end
+ end
+ end
+end
import ExUnit.CaptureLog
import Tesla.Mock
+ setup do
+ on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+ end
+
test "logs hasn't error message when request is valid" do
mock(fn
%{method: :purge, url: "http://example.com/media/example.jpg"} ->
refute capture_log(fn ->
assert Invalidation.Http.purge(
["http://example.com/media/example.jpg"],
- %{}
- ) == {:ok, "success"}
+ []
+ ) == {:ok, ["http://example.com/media/example.jpg"]}
end) =~ "Error while cache purge"
end
assert capture_log(fn ->
assert Invalidation.Http.purge(
["http://example.com/media/example1.jpg"],
- %{}
- ) == {:ok, "success"}
+ []
+ ) == {:ok, ["http://example.com/media/example1.jpg"]}
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
end
end
import ExUnit.CaptureLog
+ setup do
+ on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+ end
+
test "it logger error when script not found" do
assert capture_log(fn ->
assert Invalidation.Script.purge(
["http://example.com/media/example.jpg"],
- %{script_path: "./example"}
- ) == {:error, "\"%ErlangError{original: :enoent}\""}
- end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\""
+ script_path: "./example"
+ ) == {:error, "%ErlangError{original: :enoent}"}
+ end) =~ "Error while cache purge: %ErlangError{original: :enoent}"
- assert Invalidation.Script.purge(
- ["http://example.com/media/example.jpg"],
- %{}
- ) == {:error, "not found script path"}
+ capture_log(fn ->
+ assert Invalidation.Script.purge(
+ ["http://example.com/media/example.jpg"],
+ []
+ ) == {:error, "\"not found script path\""}
+ end)
end
end
setup do: clear_config(:media_proxy)
setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
+ setup do
+ on_exit(fn -> Cachex.clear(:deleted_urls_cache) end)
+ end
+
test "it returns 404 when MediaProxy disabled", %{conn: conn} do
Config.put([:media_proxy, :enabled], false)
assert %Plug.Conn{status: :success} = get(conn, url)
end
end
+
+ test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do
+ Config.put([:media_proxy, :enabled], true)
+ Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+ url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
+ Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png")
+
+ with_mock Pleroma.ReverseProxy,
+ call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
+ assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
+ end
+ end
end
projects / akkoma / commitdiff