Merge branch 'develop' into feature/gen-magic
authorMark Felder <feld@FreeBSD.org>
Tue, 13 Oct 2020 15:08:38 +0000 (10:08 -0500)
committerMark Felder <feld@FreeBSD.org>
Tue, 13 Oct 2020 15:08:38 +0000 (10:08 -0500)
1  2 
config/config.exs
config/description.exs
lib/pleroma/application.ex
lib/pleroma/web/activity_pub/activity_pub_controller.ex
lib/pleroma/web/mastodon_api/controllers/media_controller.ex
lib/pleroma/web/pleroma_api/controllers/account_controller.ex
lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex
test/pleroma/upload_test.exs
test/pleroma/web/activity_pub/activity_pub_controller_test.exs

diff --combined config/config.exs
index d96db7416bba6d889864bf03ddad8f98c4c7b56b,273da5bb6728faf2daf12f5bee3e66c01347d12b..2c614236033863a8560fbeae068f54eb0c24fe51
@@@ -677,7 -677,7 +677,7 @@@ config :pleroma, :rate_limit
  
  config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
  
- config :pleroma, Pleroma.Plugs.RemoteIp,
+ config :pleroma, Pleroma.Web.Plugs.RemoteIp,
    enabled: true,
    headers: ["x-forwarded-for"],
    proxies: [],
@@@ -802,8 -802,6 +802,8 @@@ config :pleroma, :hackney_pools
      timeout: 300_000
    ]
  
 +config :pleroma, :majic_pool, size: 2
 +
  private_instance? = :if_instance_is_private
  
  config :pleroma, :restrict_unauthenticated,
diff --combined config/description.exs
index f068a35de65c3c82b55ae8586ba82b5c8f321e82,6e83a8e094571e8699a857191532c5991d51c9a6..2a18989224fd12e9c0f87a16d55ba63ea4b81847
@@@ -3250,10 -3250,10 +3250,10 @@@ config :pleroma, :config_description, 
    },
    %{
      group: :pleroma,
-     key: Pleroma.Plugs.RemoteIp,
+     key: Pleroma.Web.Plugs.RemoteIp,
      type: :group,
      description: """
-     `Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
+     `Pleroma.Web.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
      **If your instance is not behind at least one reverse proxy, you should not enable this plug.**
      """,
      children: [
          ]
        }
      ]
 +  },
 +  %{
 +    group: :pleroma,
 +    key: :majic_pool,
 +    type: :group,
 +    description: "Majic/libmagic configuration",
 +    children: [
 +      %{
 +        key: :size,
 +        type: :integer,
 +        description: "Number of majic workers to start.",
 +        suggestions: [2]
 +      }
 +    ]
    }
  ]
index 4ed8df09c723c206d5eba829345665a5432f1c67,958e32db248075c33db6211e6aa4f9651b543f92..301b4e2735017efc9be8f362dadea9af25a193aa
@@@ -52,7 -52,7 +52,7 @@@ defmodule Pleroma.Application d
      Pleroma.HTML.compile_scrubbers()
      Pleroma.Config.Oban.warn()
      Config.DeprecationWarnings.warn()
-     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
+     Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
      Pleroma.ApplicationRequirements.verify!()
      setup_instrumenters()
      load_custom_modules()
          Pleroma.Repo,
          Config.TransferTask,
          Pleroma.Emoji,
-         Pleroma.Plugs.RateLimiter.Supervisor
+         Pleroma.Web.Plugs.RateLimiter.Supervisor
        ] ++
          cachex_children() ++
          http_children(adapter, @env) ++
          [
            Pleroma.Stats,
            Pleroma.JobQueueMonitor,
 +          {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
            {Oban, Config.get(Oban)}
          ] ++
          task_children(@env) ++
index 27bf6347d3ad683c9243d5034c1405b2e88eab42,6bf7421bb6e9df004577262dd22c919f0543c489..570bcc7e7384613378d0462737cc8f435a3b859f
@@@ -9,7 -9,6 +9,6 @@@ defmodule Pleroma.Web.ActivityPub.Activ
    alias Pleroma.Delivery
    alias Pleroma.Object
    alias Pleroma.Object.Fetcher
-   alias Pleroma.Plugs.EnsureAuthenticatedPlug
    alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
    alias Pleroma.Web.ActivityPub.Builder
@@@ -23,8 -22,9 +22,9 @@@
    alias Pleroma.Web.ActivityPub.Visibility
    alias Pleroma.Web.ControllerHelper
    alias Pleroma.Web.Endpoint
-   alias Pleroma.Web.FederatingPlug
    alias Pleroma.Web.Federator
+   alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
+   alias Pleroma.Web.Plugs.FederatingPlug
  
    require Logger
  
      when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
    )
  
 +  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media])
 +
    plug(
-     Pleroma.Plugs.Cache,
+     Pleroma.Web.Plugs.Cache,
      [query_params: false, tracking_fun: &__MODULE__.track_object_fetch/2]
      when action in [:activity, :object]
    )
index 09acea7f49fbab66d791da7102ca42e3760a9f2d,9586b14bc37c906d266e0c58fd41036a64ad5018..161193134132bb905ae0855a964f5bb008a273cf
@@@ -6,12 -6,11 +6,12 @@@ defmodule Pleroma.Web.MastodonAPI.Media
    use Pleroma.Web, :controller
  
    alias Pleroma.Object
-   alias Pleroma.Plugs.OAuthScopesPlug
    alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
+   alias Pleroma.Web.Plugs.OAuthScopesPlug
  
    action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
 +  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:create, :create2])
    plug(Pleroma.Web.ApiSpec.CastAndValidate)
    plug(:put_view, Pleroma.Web.MastodonAPI.StatusView)
  
index a0ed41d96708eff28d7cfb2e803acade5a7a5a55,61f4a9bd9279abbe173b8c70a60f686d1af472ea..30cf835678ca592fe8d04bd723a80fbcff26ebec
@@@ -8,20 -8,15 +8,20 @@@ defmodule Pleroma.Web.PleromaAPI.Accoun
    import Pleroma.Web.ControllerHelper,
      only: [json_response: 3, add_link_headers: 2, assign_account_by_id: 2]
  
-   alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
-   alias Pleroma.Plugs.OAuthScopesPlug
-   alias Pleroma.Plugs.RateLimiter
    alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
    alias Pleroma.Web.MastodonAPI.StatusView
+   alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
+   alias Pleroma.Web.Plugs.OAuthScopesPlug
+   alias Pleroma.Web.Plugs.RateLimiter
  
    require Pleroma.Constants
  
 +  plug(
 +    Majic.Plug,
 +    [pool: Pleroma.MajicPool] when action in [:update_avatar, :update_background, :update_banner]
 +  )
 +
    plug(
      OpenApiSpex.Plug.PutApiSpec,
      [module: Pleroma.Web.ApiSpec] when action == :confirmation_resend
index 7e2f6c3284d9b53964213f47b0008ce5f181ebef,0f6f0b9dbc6818f1efda0a2055275468fce09f1d..15210f1e65d4f4373d28876872eb8adf40bcb6b0
@@@ -5,11 -5,10 +5,11 @@@
  defmodule Pleroma.Web.PleromaAPI.MascotController do
    use Pleroma.Web, :controller
  
-   alias Pleroma.Plugs.OAuthScopesPlug
    alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
+   alias Pleroma.Web.Plugs.OAuthScopesPlug
  
 +  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:update])
    plug(Pleroma.Web.ApiSpec.CastAndValidate)
    plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action == :show)
    plug(OAuthScopesPlug, %{scopes: ["write:accounts"]} when action != :show)
  
    @doc "PUT /api/v1/pleroma/mascot"
    def update(%{assigns: %{user: user}, body_params: %{file: file}} = conn, _) do
 -    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
 -         # Reject if not an image
 -         %{type: "image"} = attachment <- render_attachment(object) do
 +    with {:content_type, "image" <> _} <- {:content_type, file.content_type},
 +         {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)) do
 +      attachment = render_attachment(object)
        {:ok, _user} = User.mascot_update(user, attachment)
  
        json(conn, attachment)
      else
 -      %{type: _} -> render_error(conn, :unsupported_media_type, "mascots can only be images")
 +      {:content_type, _} ->
 +        render_error(conn, :unsupported_media_type, "mascots can only be images")
      end
    end
  
index 4280bfcacca7147e4b05d3ef551c448346d8ebdc,b06b54487e152dc397e3127930e481cefbaedace..4280bfcacca7147e4b05d3ef551c448346d8ebdc
@@@ -11,7 -11,7 +11,7 @@@ defmodule Pleroma.UploadTest d
    alias Pleroma.Uploaders.Uploader
  
    @upload_file %Plug.Upload{
 -    content_type: "image/jpg",
 +    content_type: "image/jpeg",
      path: Path.absname("test/fixtures/image_tmp.jpg"),
      filename: "image.jpg"
    }
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: "image.jpg"
        }
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: "an [image.jpg"
        }
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: "an [image.jpg"
        }
        assert data["name"] == "an [image.jpg"
      end
  
 -    test "fixes incorrect content type" do
 -      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 -
 -      file = %Plug.Upload{
 -        content_type: "application/octet-stream",
 -        path: Path.absname("test/fixtures/image_tmp.jpg"),
 -        filename: "an [image.jpg"
 +    test "fixes incorrect content type when base64 is given" do
 +      params = %{
 +        img: "data:image/png;base64,#{Base.encode64(File.read!("test/fixtures/image.jpg"))}"
        }
  
 -      {:ok, data} = Upload.store(file, filters: [Pleroma.Upload.Filter.Dedupe])
 +      {:ok, data} = Upload.store(params)
        assert hd(data["url"])["mediaType"] == "image/jpeg"
      end
  
 -    test "adds missing extension" do
 -      File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
 -
 -      file = %Plug.Upload{
 -        content_type: "image/jpg",
 -        path: Path.absname("test/fixtures/image_tmp.jpg"),
 -        filename: "an [image"
 -      }
 -
 -      {:ok, data} = Upload.store(file)
 -      assert data["name"] == "an [image.jpg"
 -    end
 -
 -    test "fixes incorrect file extension" do
 +    test "adds extension when base64 is given" do
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
 -      file = %Plug.Upload{
 -        content_type: "image/jpg",
 -        path: Path.absname("test/fixtures/image_tmp.jpg"),
 -        filename: "an [image.blah"
 -      }
 -
 -      {:ok, data} = Upload.store(file)
 -      assert data["name"] == "an [image.jpg"
 -    end
 -
 -    test "don't modify filename of an unknown type" do
 -      File.cp("test/fixtures/test.txt", "test/fixtures/test_tmp.txt")
 -
 -      file = %Plug.Upload{
 -        content_type: "text/plain",
 -        path: Path.absname("test/fixtures/test_tmp.txt"),
 -        filename: "test.txt"
 +      params = %{
 +        img: "data:image/png;base64,#{Base.encode64(File.read!("test/fixtures/image.jpg"))}"
        }
  
 -      {:ok, data} = Upload.store(file)
 -      assert data["name"] == "test.txt"
 +      {:ok, data} = Upload.store(params)
 +      assert String.ends_with?(data["name"], ".jpg")
      end
  
      test "copies the file to the configured folder with anonymizing filename" do
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: "an [image.jpg"
        }
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: "an… image.jpg"
        }
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: ":?#[]@!$&\\'()*+,;=.jpg"
        }
        File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg")
  
        file = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "image/jpeg",
          path: Path.absname("test/fixtures/image_tmp.jpg"),
          filename: "image.jpg"
        }
index b11e2f9619defd0fe2fd42f4af3aaa000a3f9613,0517571f28f7cde9cbfa7bc36cb9fe274da313e9..b11e2f9619defd0fe2fd42f4af3aaa000a3f9613
@@@ -1487,9 -1487,9 +1487,9 @@@ defmodule Pleroma.Web.ActivityPub.Activ
        desc = "Description of the image"
  
        image = %Plug.Upload{
 -        content_type: "image/jpg",
 +        content_type: "bad/content-type",
          path: Path.absname("test/fixtures/image.jpg"),
 -        filename: "an_image.jpg"
 +        filename: "an_image.png"
        }
  
        object =
        assert [%{"href" => object_href, "mediaType" => object_mediatype}] = object["url"]
        assert is_binary(object_href)
        assert object_mediatype == "image/jpeg"
 +      assert String.ends_with?(object_href, ".jpg")
  
        activity_request = %{
          "@context" => "https://www.w3.org/ns/activitystreams",