From: feld Date: Fri, 18 Sep 2020 18:38:20 +0000 (+0000) Subject: Merge branch 'media-preview-proxy-nostream' into 'develop' X-Git-Url: http://git.squeep.com/?a=commitdiff_plain;h=6c052bd5b6cc29d321b500654bd6b098d0e6c56a;hp=-c;p=akkoma Merge branch 'media-preview-proxy-nostream' into 'develop' Media preview proxy See merge request pleroma/pleroma!3001 --- 6c052bd5b6cc29d321b500654bd6b098d0e6c56a diff --combined CHANGELOG.md index 5e4d06c82,497fb62f3..5a7e27fd3 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -5,9 -5,6 +5,9 @@@ The format is based on [Keep a Changelo ## Unreleased +### Added +- Experimental websocket-based federation between Pleroma instances. + ### Changed - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. @@@ -15,6 -12,10 +15,10 @@@ - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false. - Users with the `discoverable` field set to false will not show up in searches. - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option). + + ### Added + - Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details). + ### Removed - **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation). @@@ -23,27 -24,18 +27,27 @@@ - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were switched to a new configuration mechanism, however it was not officially removed until now. -## unreleased-patch - ??? +## [2.1.2] - 2020-09-17 + +### Security + +- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event). ### Fixed -- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance -- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user +- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance. +- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user. +- Mastodon Streaming API: Handler crashes on authentication failures, resulting in error logs. +- Mastodon Streaming API: Error logs on client pings. +- Rich media: Log spam on failures. Now the error is only logged once per attempt. + +### Changed -## Unreleased-patch +- Rich Media: A HEAD request is now done to the url, to ensure it has the appropriate content type and size before proceeding with a GET. -### Security +### Upgrade notes -- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event) +1. Restart Pleroma ## [2.1.1] - 2020-09-08 diff --combined config/config.exs index 147f09722,98c31ef86..00624bf00 --- a/config/config.exs +++ b/config/config.exs @@@ -130,7 -130,6 +130,7 @@@ config :pleroma, Pleroma.Web.Endpoint dispatch: [ {:_, [ + {"/api/fedsocket/v1", Pleroma.Web.FedSockets.IncomingHandler, []}, {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []}, {"/websocket", Phoenix.Endpoint.CowboyWebSocket, {Phoenix.Transports.WebSocket, @@@ -149,16 -148,6 +149,16 @@@ "SameSite=Lax" ] +config :pleroma, :fed_sockets, + enabled: false, + connection_duration: :timer.hours(8), + rejection_duration: :timer.minutes(15), + fed_socket_fetches: [ + default: 12_000, + interval: 3_000, + lazy: false + ] + # Configures Elixir's Logger config :logger, :console, level: :debug, @@@ -434,6 -423,8 +434,8 @@@ config :pleroma, :media_proxy proxy_opts: [ redirect_on_failure: false, max_body_length: 25 * 1_048_576, + # Note: max_read_duration defaults to Pleroma.ReverseProxy.max_read_duration_default/1 + max_read_duration: 30_000, http: [ follow_redirect: true, pool: :media @@@ -448,6 -439,14 +450,14 @@@ config :pleroma, Pleroma.Web.MediaProxy config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil + # Note: media preview proxy depends on media proxy to be enabled + config :pleroma, :media_preview_proxy, + enabled: false, + thumbnail_max_width: 600, + thumbnail_max_height: 600, + image_quality: 85, + min_content_length: 100 * 1024 + config :pleroma, :chat, enabled: true config :phoenix, :format_encoders, json: Jason @@@ -543,7 -542,6 +553,7 @@@ config :pleroma, Oban token_expiration: 5, federator_incoming: 50, federator_outgoing: 50, + ingestion_queue: 50, web_push: 50, mailer: 10, transmogrifier: 20, @@@ -753,8 -751,8 +763,8 @@@ config :pleroma, :pools ], media: [ size: 50, - max_waiting: 10, - recv_timeout: 10_000 + max_waiting: 20, + recv_timeout: 15_000 ], upload: [ size: 25, @@@ -799,8 -797,6 +809,8 @@@ config :tzdata, :http_client, Pleroma.H config :ex_aws, http_client: Pleroma.HTTP.ExAws +config :web_push_encryption, http_client: Pleroma.HTTP + config :pleroma, :instances_favicons, enabled: false config :floki, :html_parser, Floki.HTMLParser.FastHtml diff --combined config/description.exs index 6f3855918,4a5d5f2ea..097e98633 --- a/config/description.exs +++ b/config/description.exs @@@ -270,19 -270,6 +270,19 @@@ config :pleroma, :config_description, } ] }, + %{ + group: :pleroma, + key: :fed_sockets, + type: :group, + description: "Websocket based federation", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enable FedSockets" + } + ] + }, %{ group: :pleroma, key: Pleroma.Emails.Mailer, @@@ -1887,6 -1874,7 +1887,7 @@@ suggestions: [ redirect_on_failure: false, max_body_length: 25 * 1_048_576, + max_read_duration: 30_000, http: [ follow_redirect: true, pool: :media @@@ -1907,6 -1895,11 +1908,11 @@@ "Limits the content length to be approximately the " <> "specified length. It is validated with the `content-length` header and also verified when proxying." }, + %{ + key: :max_read_duration, + type: :integer, + description: "Timeout (in milliseconds) of GET request to remote URI." + }, %{ key: :http, label: "HTTP", @@@ -1953,6 -1946,43 +1959,43 @@@ } ] }, + %{ + group: :pleroma, + key: :media_preview_proxy, + type: :group, + description: "Media preview proxy", + children: [ + %{ + key: :enabled, + type: :boolean, + description: + "Enables proxying of remote media preview to the instance's proxy. Requires enabled media proxy." + }, + %{ + key: :thumbnail_max_width, + type: :integer, + description: + "Max width of preview thumbnail for images (video preview always has original dimensions)." + }, + %{ + key: :thumbnail_max_height, + type: :integer, + description: + "Max height of preview thumbnail for images (video preview always has original dimensions)." + }, + %{ + key: :image_quality, + type: :integer, + description: "Quality of the output. Ranges from 0 (min quality) to 100 (max quality)." + }, + %{ + key: :min_content_length, + type: :integer, + description: + "Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing." + } + ] + }, %{ group: :pleroma, key: Pleroma.Web.MediaProxy.Invalidation.Http, diff --combined docs/configuration/cheatsheet.md index 9a275294e,d7c342383..42e5fe808 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@@ -225,16 -225,6 +225,16 @@@ Enables the worker which processes post * `enabled`: whether expired activities will be sent to the job queue to be deleted +## FedSockets +FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options. + +### :fedsockets +* `enabled`: Enables FedSockets for this instance. `false` by default. +* `connection_duration`: Time an idle websocket is kept open. +* `rejection_duration`: Failures to connect via FedSockets will not be retried for this period of time. +* `fed_socket_fetches` and `fed_socket_rejections`: Settings passed to `cachex` for the fetch registry, and rejection stacks. See `Pleroma.Web.FedSockets` for more details. + + ## Frontends ### :frontend_configurations @@@ -324,6 -314,14 +324,14 @@@ This section describe PWA manifest inst * `enabled`: Enables purge cache * `provider`: Which one of the [purge cache strategy](#purge-cache-strategy) to use. + ## :media_preview_proxy + + * `enabled`: Enables proxying of remote media preview to the instance’s proxy. Requires enabled media proxy (`media_proxy/enabled`). + * `thumbnail_max_width`: Max width of preview thumbnail for images (video preview always has original dimensions). + * `thumbnail_max_height`: Max height of preview thumbnail for images (video preview always has original dimensions). + * `image_quality`: Quality of the output. Ranges from 0 (min quality) to 100 (max quality). + * `min_content_length`: Min content length to perform preview, in bytes. If greater than 0, media smaller in size will be served as is, without thumbnailing. + ### Purge cache strategy #### Pleroma.Web.MediaProxy.Invalidation.Script diff --combined mix.lock index a17e8c0fc,1ab0934c8..adb3f024a --- a/mix.lock +++ b/mix.lock @@@ -31,6 -31,7 +31,7 @@@ "ecto": {:hex, :ecto, "3.4.5", "2bcd262f57b2c888b0bd7f7a28c8a48aa11dc1a2c6a858e45dd8f8426d504265", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8c6d1d4d524559e9b7a062f0498e2c206122552d63eacff0a6567ffe7a8e8691"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"}, + "eimp": {:hex, :eimp, "1.0.14", "fc297f0c7e2700457a95a60c7010a5f1dcb768a083b6d53f49cd94ab95a28f22", [:rebar3], [{:p1_utils, "1.0.18", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm", "501133f3112079b92d9e22da8b88bf4f0e13d4d67ae9c15c42c30bd25ceb83b6"}, "elixir_make": {:hex, :elixir_make, "0.6.0", "38349f3e29aff4864352084fc736fa7fa0f2995a819a737554f7ebd28b85aaab", [:mix], [], "hexpm", "d522695b93b7f0b4c0fcb2dfe73a6b905b1c301226a5a55cb42e5b14d509e050"}, "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"}, @@@ -80,6 -81,7 +81,7 @@@ "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"}, "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]}, + "p1_utils": {:hex, :p1_utils, "1.0.18", "3fe224de5b2e190d730a3c5da9d6e8540c96484cf4b4692921d1e28f0c32b01c", [:rebar3], [], "hexpm", "1fc8773a71a15553b179c986b22fbeead19b28fe486c332d4929700ffeb71f88"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "1.2.1", "9cbe354b58121075bd20eb83076900a3832324b7dd171a6895fab57b6bb2752c", [:mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}], "hexpm", "d3b40a4a4630f0b442f19eca891fcfeeee4c40871936fed2f68e1c4faa30481f"}, "phoenix": {:hex, :phoenix, "1.4.17", "1b1bd4cff7cfc87c94deaa7d60dd8c22e04368ab95499483c50640ef3bd838d8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a8e5d7a3d76d452bb5fb86e8b7bd115f737e4f8efe202a463d4aeb4a5809611"}, @@@ -118,5 -120,5 +120,5 @@@ "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, "web_push_encryption": {:hex, :web_push_encryption, "0.3.0", "598b5135e696fd1404dc8d0d7c0fa2c027244a4e5d5e5a98ba267f14fdeaabc8", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "f10bdd1afe527ede694749fb77a2f22f146a51b054c7fa541c9fd920fba7c875"}, - "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, + "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []} }