From: Mark Felder Date: Tue, 13 Oct 2020 15:08:38 +0000 (-0500) Subject: Merge branch 'develop' into feature/gen-magic X-Git-Url: http://git.squeep.com/?a=commitdiff_plain;h=b738f709532ff18845f5d8cc3888d0bd67f750ab;hp=-c;p=akkoma Merge branch 'develop' into feature/gen-magic --- b738f709532ff18845f5d8cc3888d0bd67f750ab diff --combined config/config.exs index d96db7416,273da5bb6..2c6142360 --- a/config/config.exs +++ b/config/config.exs @@@ -677,7 -677,7 +677,7 @@@ config :pleroma, :rate_limit config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600 - config :pleroma, Pleroma.Plugs.RemoteIp, + config :pleroma, Pleroma.Web.Plugs.RemoteIp, enabled: true, headers: ["x-forwarded-for"], proxies: [], @@@ -802,8 -802,6 +802,8 @@@ config :pleroma, :hackney_pools timeout: 300_000 ] +config :pleroma, :majic_pool, size: 2 + private_instance? = :if_instance_is_private config :pleroma, :restrict_unauthenticated, diff --combined config/description.exs index f068a35de,6e83a8e09..2a1898922 --- a/config/description.exs +++ b/config/description.exs @@@ -3250,10 -3250,10 +3250,10 @@@ config :pleroma, :config_description, }, %{ group: :pleroma, - key: Pleroma.Plugs.RemoteIp, + key: Pleroma.Web.Plugs.RemoteIp, type: :group, description: """ - `Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. + `Pleroma.Web.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. **If your instance is not behind at least one reverse proxy, you should not enable this plug.** """, children: [ @@@ -3708,19 -3708,5 +3708,19 @@@ ] } ] + }, + %{ + group: :pleroma, + key: :majic_pool, + type: :group, + description: "Majic/libmagic configuration", + children: [ + %{ + key: :size, + type: :integer, + description: "Number of majic workers to start.", + suggestions: [2] + } + ] } ] diff --combined lib/pleroma/application.ex index 4ed8df09c,958e32db2..301b4e273 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@@ -52,7 -52,7 +52,7 @@@ defmodule Pleroma.Application d Pleroma.HTML.compile_scrubbers() Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() + Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.ApplicationRequirements.verify!() setup_instrumenters() load_custom_modules() @@@ -88,14 -88,13 +88,14 @@@ Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, - Pleroma.Plugs.RateLimiter.Supervisor + Pleroma.Web.Plugs.RateLimiter.Supervisor ] ++ cachex_children() ++ http_children(adapter, @env) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, + {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]}, {Oban, Config.get(Oban)} ] ++ task_children(@env) ++ diff --combined lib/pleroma/web/activity_pub/activity_pub_controller.ex index 27bf6347d,6bf7421bb..570bcc7e7 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@@ -9,7 -9,6 +9,6 @@@ defmodule Pleroma.Web.ActivityPub.Activ alias Pleroma.Delivery alias Pleroma.Object alias Pleroma.Object.Fetcher - alias Pleroma.Plugs.EnsureAuthenticatedPlug alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder @@@ -23,8 -22,9 +22,9 @@@ alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.ControllerHelper alias Pleroma.Web.Endpoint - alias Pleroma.Web.FederatingPlug alias Pleroma.Web.Federator + alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug + alias Pleroma.Web.Plugs.FederatingPlug require Logger @@@ -45,10 -45,8 +45,10 @@@ when action in [:read_inbox, :update_outbox, :whoami, :upload_media] ) + plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media]) + plug( - Pleroma.Plugs.Cache, + Pleroma.Web.Plugs.Cache, [query_params: false, tracking_fun: &__MODULE__.track_object_fetch/2] when action in [:activity, :object] ) diff --combined lib/pleroma/web/mastodon_api/controllers/media_controller.ex index 09acea7f4,9586b14bc..161193134 --- a/lib/pleroma/web/mastodon_api/controllers/media_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/media_controller.ex @@@ -6,12 -6,11 +6,12 @@@ defmodule Pleroma.Web.MastodonAPI.Media use Pleroma.Web, :controller alias Pleroma.Object - alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.Plugs.OAuthScopesPlug action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:create, :create2]) plug(Pleroma.Web.ApiSpec.CastAndValidate) plug(:put_view, Pleroma.Web.MastodonAPI.StatusView) diff --combined lib/pleroma/web/pleroma_api/controllers/account_controller.ex index a0ed41d96,61f4a9bd9..30cf83567 --- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex @@@ -8,20 -8,15 +8,20 @@@ defmodule Pleroma.Web.PleromaAPI.Accoun import Pleroma.Web.ControllerHelper, only: [json_response: 3, add_link_headers: 2, assign_account_by_id: 2] - alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug - alias Pleroma.Plugs.OAuthScopesPlug - alias Pleroma.Plugs.RateLimiter alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug + alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.Web.Plugs.RateLimiter require Pleroma.Constants + plug( + Majic.Plug, + [pool: Pleroma.MajicPool] when action in [:update_avatar, :update_background, :update_banner] + ) + plug( OpenApiSpex.Plug.PutApiSpec, [module: Pleroma.Web.ApiSpec] when action == :confirmation_resend diff --combined lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex index 7e2f6c328,0f6f0b9db..15210f1e6 --- a/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex @@@ -5,11 -5,10 +5,11 @@@ defmodule Pleroma.Web.PleromaAPI.MascotController do use Pleroma.Web, :controller - alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.Plugs.OAuthScopesPlug + plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:update]) plug(Pleroma.Web.ApiSpec.CastAndValidate) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action == :show) plug(OAuthScopesPlug, %{scopes: ["write:accounts"]} when action != :show) @@@ -23,15 -22,14 +23,15 @@@ @doc "PUT /api/v1/pleroma/mascot" def update(%{assigns: %{user: user}, body_params: %{file: file}} = conn, _) do - with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)), - # Reject if not an image - %{type: "image"} = attachment <- render_attachment(object) do + with {:content_type, "image" <> _} <- {:content_type, file.content_type}, + {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)) do + attachment = render_attachment(object) {:ok, _user} = User.mascot_update(user, attachment) json(conn, attachment) else - %{type: _} -> render_error(conn, :unsupported_media_type, "mascots can only be images") + {:content_type, _} -> + render_error(conn, :unsupported_media_type, "mascots can only be images") end end diff --combined test/pleroma/upload_test.exs index 4280bfcac,b06b54487..4280bfcac --- a/test/pleroma/upload_test.exs +++ b/test/pleroma/upload_test.exs @@@ -11,7 -11,7 +11,7 @@@ defmodule Pleroma.UploadTest d alias Pleroma.Uploaders.Uploader @upload_file %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "image.jpg" } @@@ -124,7 -124,7 +124,7 @@@ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "image.jpg" } @@@ -140,7 -140,7 +140,7 @@@ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "an [image.jpg" } @@@ -156,7 -156,7 +156,7 @@@ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "an [image.jpg" } @@@ -165,31 -165,63 +165,31 @@@ assert data["name"] == "an [image.jpg" end - test "fixes incorrect content type" do - File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - - file = %Plug.Upload{ - content_type: "application/octet-stream", - path: Path.absname("test/fixtures/image_tmp.jpg"), - filename: "an [image.jpg" + test "fixes incorrect content type when base64 is given" do + params = %{ + img: "data:image/png;base64,#{Base.encode64(File.read!("test/fixtures/image.jpg"))}" } - {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe]) + {:ok, data} = Upload.store(params) assert hd(data["url"])["mediaType"] == "image/jpeg" end - test "adds missing extension" do - File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - - file = %Plug.Upload{ - content_type: "image/jpg", - path: Path.absname("test/fixtures/image_tmp.jpg"), - filename: "an [image" - } - - {:ok, data} = Upload.store(file) - assert data["name"] == "an [image.jpg" - end - - test "fixes incorrect file extension" do + test "adds extension when base64 is given" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - file = %Plug.Upload{ - content_type: "image/jpg", - path: Path.absname("test/fixtures/image_tmp.jpg"), - filename: "an [image.blah" - } - - {:ok, data} = Upload.store(file) - assert data["name"] == "an [image.jpg" - end - - test "don't modify filename of an unknown type" do - File.cp("test/fixtures/test.txt", "test/fixtures/test_tmp.txt") - - file = %Plug.Upload{ - content_type: "text/plain", - path: Path.absname("test/fixtures/test_tmp.txt"), - filename: "test.txt" + params = %{ + img: "data:image/png;base64,#{Base.encode64(File.read!("test/fixtures/image.jpg"))}" } - {:ok, data} = Upload.store(file) - assert data["name"] == "test.txt" + {:ok, data} = Upload.store(params) + assert String.ends_with?(data["name"], ".jpg") end test "copies the file to the configured folder with anonymizing filename" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "an [image.jpg" } @@@ -203,7 -235,7 +203,7 @@@ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "an… image.jpg" } @@@ -218,7 -250,7 +218,7 @@@ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: ":?#[]@!$&\\'()*+,;=.jpg" } @@@ -240,7 -272,7 +240,7 @@@ File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") file = %Plug.Upload{ - content_type: "image/jpg", + content_type: "image/jpeg", path: Path.absname("test/fixtures/image_tmp.jpg"), filename: "image.jpg" } diff --combined test/pleroma/web/activity_pub/activity_pub_controller_test.exs index b11e2f961,0517571f2..b11e2f961 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@@ -1487,9 -1487,9 +1487,9 @@@ defmodule Pleroma.Web.ActivityPub.Activ desc = "Description of the image" image = %Plug.Upload{ - content_type: "image/jpg", + content_type: "bad/content-type", path: Path.absname("test/fixtures/image.jpg"), - filename: "an_image.jpg" + filename: "an_image.png" } object = @@@ -1504,7 -1504,6 +1504,7 @@@ assert [%{"href" => object_href, "mediaType" => object_mediatype}] = object["url"] assert is_binary(object_href) assert object_mediatype == "image/jpeg" + assert String.ends_with?(object_href, ".jpg") activity_request = %{ "@context" => "https://www.w3.org/ns/activitystreams",