Use multiple hackney pools

* federation (ap, salmon)
* media (rich media, media proxy)
* upload (uploader proxy)

Each "part" will stop fighting others ones -- a huge federation outbound
could before make the media proxy fail to checkout a connection in time.

splitted media and uploaded media for the good reason than an upload
pool will have all connections to the same host (the uploader upstream).
it also has a longer default retention period for connections.

diff --git a/config/config.exs b/config/config.exs
index 1180a50bcd068703d10a792bb936b1d4352daa48..4dc7a62d72824b84fb813c46b73295d1274a73fc 100644 (file)
--- a/config/config.exs
+++ b/config/config.exs
@@ -15,6 +15,20 @@ config :pleroma, Pleroma.Captcha,
   seconds_valid: 60,
   method: Pleroma.Captcha.Kocaptcha
 
+config :pleroma, :hackney_pools,
+  federation: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  media: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  upload: [
+    max_connections: 25,
+    timeout: 300_000
+  ]
+
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
 
 # Upload configuration
@@ -22,7 +36,14 @@ config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
   filters: [],
   proxy_remote: false,
-  proxy_opts: []
+  proxy_opts: [
+    redirect_on_failure: false,
+    max_body_length: 25 * 1_048_576,
+    http: [
+      follow_redirect: true,
+      pool: :upload
+    ]
+  ]
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
@@ -214,7 +235,16 @@ config :pleroma, :mrf_simple,
   reject: [],
   accept: []
 
-config :pleroma, :media_proxy, enabled: false
+config :pleroma, :media_proxy,
+  enabled: false,
+  proxy_opts: [
+    redirect_on_failure: false,
+    max_body_length: 25 * 1_048_576,
+    http: [
+      follow_redirect: true,
+      pool: :media
+    ]
+  ]
 
 config :pleroma, :chat, enabled: true
 

diff --git a/docs/config.md b/docs/config.md
index 8740c3fae81f267bf4078c9e8f1dadec186eceb5..a00532d16f41eec1612cc08226fbd339b1bf7aba 100644 (file)
--- a/docs/config.md
+++ b/docs/config.md
@@ -234,3 +234,20 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando
   * Pleroma.Web.Metadata.Providers.OpenGraph
   * Pleroma.Web.Metadata.Providers.TwitterCard
 * `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews
+
+## :hackney_pools
+
+Advanced. Tweaks Hackney (http client) connections pools.
+
+There's three pools used:
+
+* `:federation` for the federation jobs.
+  You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
+* `:media` for rich media, media proxy
+* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
+
+For each pool, the options are:
+
+* `max_connections` - how much connections a pool can hold
+* `timeout` - retention duration for connections
+

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index c65bebb3b98a0a3cffb462a87d3931340de7c9f2..f775cd53dca89d97580ec78bd275d96e5d4c1747 100644 (file)
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -101,12 +101,15 @@ defmodule Pleroma.Application do
           ],
           id: :cachex_idem
         ),
-        worker(Pleroma.FlakeId, []),
-        worker(Pleroma.Web.Federator.RetryQueue, []),
-        worker(Pleroma.Web.Federator, []),
-        worker(Pleroma.Stats, []),
-        worker(Pleroma.Web.Push, [])
+        worker(Pleroma.FlakeId, [])
       ] ++
+        hackney_pool_children() ++
+        [
+          worker(Pleroma.Web.Federator.RetryQueue, []),
+          worker(Pleroma.Web.Federator, []),
+          worker(Pleroma.Stats, []),
+          worker(Pleroma.Web.Push, [])
+        ] ++
         streamer_child() ++
         chat_child() ++
         [
@@ -121,6 +124,20 @@ defmodule Pleroma.Application do
     Supervisor.start_link(children, opts)
   end
 
+  def enabled_hackney_pools() do
+    [:media] ++
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        [:federation]
+      else
+        []
+      end ++
+      if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do
+        [:uploadproxy]
+      else
+        []
+      end
+  end
+
   if Mix.env() == :test do
     defp streamer_child(), do: []
     defp chat_child(), do: []
@@ -137,4 +154,11 @@ defmodule Pleroma.Application do
       end
     end
   end
+
+  defp hackney_pool_children() do
+    for pool <- enabled_hackney_pools() do
+      options = Pleroma.Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
 end

diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex
index 699d80cd7a7d24f8f3782f2d81742276fc4eb4ac..b798eaa5ac63e54dc079b9ed20732f85d0a9600e 100644 (file)
--- a/lib/pleroma/http/connection.ex
+++ b/lib/pleroma/http/connection.ex
@@ -10,7 +10,8 @@ defmodule Pleroma.HTTP.Connection do
   @hackney_options [
     timeout: 10000,
     recv_timeout: 20000,
-    follow_redirect: true
+    follow_redirect: true,
+    pool: :federation
   ]
   @adapter Application.get_env(:tesla, :adapter)
 

diff --git a/lib/pleroma/uploaders/mdii.ex b/lib/pleroma/uploaders/mdii.ex
index 530b34362b8d7936b6bb5b59262b77f25f2f1b01..320b07abde423dc597cd0585161002223854c514 100644 (file)
--- a/lib/pleroma/uploaders/mdii.ex
+++ b/lib/pleroma/uploaders/mdii.ex
@@ -24,7 +24,8 @@ defmodule Pleroma.Uploaders.MDII do
     extension = String.split(upload.name, ".") |> List.last()
     query = "#{cgi}?#{extension}"
 
-    with {:ok, %{status: 200, body: body}} <- @httpoison.post(query, file_data) do
+    with {:ok, %{status: 200, body: body}} <-
+           @httpoison.post(query, file_data, adapter: [pool: :default]) do
       remote_file_name = String.split(body) |> List.first()
       public_url = "#{files}/#{remote_file_name}.#{extension}"
       {:ok, {:url, public_url}}

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 65b61202624a2bb76bb4ac9862074960901516ab..a92645ca30386cc45a3471b6936f26513d512163 100644 (file)
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1311,7 +1311,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
                [],
                adapter: [
                  timeout: timeout,
-                 recv_timeout: timeout
+                 recv_timeout: timeout,
+                 pool: :default
                ]
              ),
            {:ok, data} <- Jason.decode(body) do

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 76d977ac22a68f02a5f841f072fe998a161211cb..874e8c5e68eea6313fdbce0d4c9eec4a2ece8073 100644 (file)
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
 
   defp parse_url(url) do
     try do
-      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url)
+      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], pool: :media)
 
       html |> maybe_parse() |> get_parsed_data()
     rescue

diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
index efa98bc2c16fe47a4c788cb663e28c13dcf88cf9..c3adc4962e177c72f86063d16bc127e7d59be90c 100644 (file)
--- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
@@ -20,7 +20,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
   end
 
   defp get_oembed_data(url) do
-    {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url)
+    {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url, [], pool: :media)
 
     {:ok, data} = Jason.decode(json)