Merge remote-tracking branch 'remotes/origin/develop' into output-of-relationships...
authorIvan Tashkinov <ivantashkinov@gmail.com>
Wed, 1 Apr 2020 17:08:14 +0000 (20:08 +0300)
committerIvan Tashkinov <ivantashkinov@gmail.com>
Wed, 1 Apr 2020 17:08:14 +0000 (20:08 +0300)
# Conflicts:
# CHANGELOG.md

1  2 
CHANGELOG.md
config/config.exs
config/description.exs
lib/mix/tasks/pleroma/benchmark.ex
test/web/mastodon_api/controllers/notification_controller_test.exs

diff --combined CHANGELOG.md
index a391bf1fa2ffdc45869f9ca33031ba30b33b6f79,52e6c33f81ffa9e6e29cfdb39722b2bd982a4cd4..b41502a272e736cfc98f94fd57b4b3a00cb31284
@@@ -13,7 -13,7 +13,8 @@@ The format is based on [Keep a Changelo
  ### Added
  - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
  - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
+ - New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
 +- Configuration: `:extensions/:output_relationships_in_statuses_by_default` option (if `false`, disables the output of account/pleroma/relationship for statuses and notifications by default, breaking the compatibility with older PleromaFE versions).
  <details>
    <summary>API Changes</summary>
  - Mastodon API: Support for `include_types` in `/api/v1/notifications`.
@@@ -21,7 -21,7 +22,7 @@@
  
  ## [2.0.0] - 2019-03-08
  ### Security
 -- Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
 +- Mastodon API: Fix being able to request enormous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
  
  ### Removed
  - **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media`
diff --combined config/config.exs
index 73bf658fe07c1d02e405551057dabe7b60f7c6e2,232a91bf132c8063f8f0f730121bb68eade6ad6f..7f013aaad71f65d279b7f1bdb027370f17c6c0c4
@@@ -58,20 -58,6 +58,6 @@@ config :pleroma, Pleroma.Captcha
  
  config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
  
- config :pleroma, :hackney_pools,
-   federation: [
-     max_connections: 50,
-     timeout: 150_000
-   ],
-   media: [
-     max_connections: 50,
-     timeout: 150_000
-   ],
-   upload: [
-     max_connections: 25,
-     timeout: 300_000
-   ]
  # Upload configuration
  config :pleroma, Pleroma.Upload,
    uploader: Pleroma.Uploaders.Local,
@@@ -184,21 -170,13 +170,13 @@@ config :mime, :types, %
    "application/ld+json" => ["activity+json"]
  }
  
- config :tesla, adapter: Tesla.Adapter.Hackney
+ config :tesla, adapter: Tesla.Adapter.Gun
  # Configures http settings, upstream proxy etc.
  config :pleroma, :http,
    proxy_url: nil,
    send_user_agent: true,
    user_agent: :default,
-   adapter: [
-     ssl_options: [
-       # Workaround for remote server certificate chain issues
-       partial_chain: &:hackney_connect.partial_chain/1,
-       # We don't support TLS v1.3 yet
-       versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
-     ]
-   ]
+   adapter: []
  
  config :pleroma, :instance,
    name: "Pleroma",
    extended_nickname_format: true,
    cleanup_attachments: false
  
 +config :pleroma, :extensions, output_relationships_in_statuses_by_default: true
 +
  config :pleroma, :feed,
    post_title: %{
      max_length: 100,
@@@ -626,6 -602,49 +604,49 @@@ config :pleroma, Pleroma.Repo
    parameters: [gin_fuzzy_search_limit: "500"],
    prepare: :unnamed
  
+ config :pleroma, :connections_pool,
+   checkin_timeout: 250,
+   max_connections: 250,
+   retry: 1,
+   retry_timeout: 1000,
+   await_up_timeout: 5_000
+ config :pleroma, :pools,
+   federation: [
+     size: 50,
+     max_overflow: 10,
+     timeout: 150_000
+   ],
+   media: [
+     size: 50,
+     max_overflow: 10,
+     timeout: 150_000
+   ],
+   upload: [
+     size: 25,
+     max_overflow: 5,
+     timeout: 300_000
+   ],
+   default: [
+     size: 10,
+     max_overflow: 2,
+     timeout: 10_000
+   ]
+ config :pleroma, :hackney_pools,
+   federation: [
+     max_connections: 50,
+     timeout: 150_000
+   ],
+   media: [
+     max_connections: 50,
+     timeout: 150_000
+   ],
+   upload: [
+     max_connections: 25,
+     timeout: 300_000
+   ]
  config :pleroma, :restrict_unauthenticated,
    timelines: %{local: false, federated: false},
    profiles: %{local: false, remote: false},
diff --combined config/description.exs
index d127f8f203a3d46c43911e9f94f880e153f74567,642f1a3ce9d2be466a15fe24495dfb189bf8ed0b..1b450db58de2465456fbbd2a63489f734ea42109
@@@ -121,22 -121,6 +121,22 @@@ config :pleroma, :config_description, 
        }
      ]
    },
 +  %{
 +    group: :pleroma,
 +    key: :extensions,
 +    type: :group,
 +    description: "Pleroma-specific extensions",
 +    children: [
 +      %{
 +        key: :output_relationships_in_statuses_by_default,
 +        type: :beeolean,
 +        description:
 +          "If `true`, outputs account/pleroma/relationship map for each rendered status / notification (for all clients). " <>
 +            "If `false`, outputs the above only if `with_relationships` param is tru-ish " <>
 +            "(that breaks compatibility with older PleromaFE versions which do not send this param but expect the output)."
 +      }
 +    ]
 +  },
    %{
      group: :pleroma,
      key: Pleroma.Uploaders.Local,
        }
      ]
    },
+   %{
+     group: :pleroma,
+     key: :connections_pool,
+     type: :group,
+     description: "Advanced settings for `gun` connections pool",
+     children: [
+       %{
+         key: :checkin_timeout,
+         type: :integer,
+         description: "Timeout to checkin connection from pool. Default: 250ms.",
+         suggestions: [250]
+       },
+       %{
+         key: :max_connections,
+         type: :integer,
+         description: "Maximum number of connections in the pool. Default: 250 connections.",
+         suggestions: [250]
+       },
+       %{
+         key: :retry,
+         type: :integer,
+         description:
+           "Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.",
+         suggestions: [1]
+       },
+       %{
+         key: :retry_timeout,
+         type: :integer,
+         description:
+           "Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.",
+         suggestions: [1000]
+       },
+       %{
+         key: :await_up_timeout,
+         type: :integer,
+         description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
+         suggestions: [5000]
+       }
+     ]
+   },
+   %{
+     group: :pleroma,
+     key: :pools,
+     type: :group,
+     description: "Advanced settings for `gun` workers pools",
+     children: [
+       %{
+         key: :federation,
+         type: :keyword,
+         description: "Settings for federation pool.",
+         children: [
+           %{
+             key: :size,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [50]
+           },
+           %{
+             key: :max_overflow,
+             type: :integer,
+             description: "Number of additional workers if pool is under load.",
+             suggestions: [10]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `gun` will wait for response.",
+             suggestions: [150_000]
+           }
+         ]
+       },
+       %{
+         key: :media,
+         type: :keyword,
+         description: "Settings for media pool.",
+         children: [
+           %{
+             key: :size,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [50]
+           },
+           %{
+             key: :max_overflow,
+             type: :integer,
+             description: "Number of additional workers if pool is under load.",
+             suggestions: [10]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `gun` will wait for response.",
+             suggestions: [150_000]
+           }
+         ]
+       },
+       %{
+         key: :upload,
+         type: :keyword,
+         description: "Settings for upload pool.",
+         children: [
+           %{
+             key: :size,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [25]
+           },
+           %{
+             key: :max_overflow,
+             type: :integer,
+             description: "Number of additional workers if pool is under load.",
+             suggestions: [5]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `gun` will wait for response.",
+             suggestions: [300_000]
+           }
+         ]
+       },
+       %{
+         key: :default,
+         type: :keyword,
+         description: "Settings for default pool.",
+         children: [
+           %{
+             key: :size,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [10]
+           },
+           %{
+             key: :max_overflow,
+             type: :integer,
+             description: "Number of additional workers if pool is under load.",
+             suggestions: [2]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `gun` will wait for response.",
+             suggestions: [10_000]
+           }
+         ]
+       }
+     ]
+   },
+   %{
+     group: :pleroma,
+     key: :hackney_pools,
+     type: :group,
+     description: "Advanced settings for `hackney` connections pools",
+     children: [
+       %{
+         key: :federation,
+         type: :keyword,
+         description: "Settings for federation pool.",
+         children: [
+           %{
+             key: :max_connections,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [50]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `hackney` will wait for response.",
+             suggestions: [150_000]
+           }
+         ]
+       },
+       %{
+         key: :media,
+         type: :keyword,
+         description: "Settings for media pool.",
+         children: [
+           %{
+             key: :max_connections,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [50]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `hackney` will wait for response.",
+             suggestions: [150_000]
+           }
+         ]
+       },
+       %{
+         key: :upload,
+         type: :keyword,
+         description: "Settings for upload pool.",
+         children: [
+           %{
+             key: :max_connections,
+             type: :integer,
+             description: "Number workers in the pool.",
+             suggestions: [25]
+           },
+           %{
+             key: :timeout,
+             type: :integer,
+             description: "Timeout while `hackney` will wait for response.",
+             suggestions: [300_000]
+           }
+         ]
+       }
+     ]
+   },
    %{
      group: :pleroma,
      key: :restrict_unauthenticated,
index b2bbe40ac7aca04a600d274df8faa446b6dd5dd9,dd2b9c8f278b26d28506b14a413d341a704c6417..6ab7fe8ef6aaac665ab330b4c3b3fbaa75671140
@@@ -67,12 -67,50 +67,51 @@@ defmodule Mix.Tasks.Pleroma.Benchmark d
            Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
              activities: activities,
              for: user,
 -            as: :activity
 +            as: :activity,
 +            skip_relationships: true
            })
          end
        },
        inputs: inputs
      )
    end
+   def run(["adapters"]) do
+     start_pleroma()
+     :ok =
+       Pleroma.Gun.Conn.open(
+         "https://httpbin.org/stream-bytes/1500",
+         :gun_connections
+       )
+     Process.sleep(1_500)
+     Benchee.run(
+       %{
+         "Without conn and without pool" => fn ->
+           {:ok, %Tesla.Env{}} =
+             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+               adapter: [pool: :no_pool, receive_conn: false]
+             )
+         end,
+         "Without conn and with pool" => fn ->
+           {:ok, %Tesla.Env{}} =
+             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+               adapter: [receive_conn: false]
+             )
+         end,
+         "With reused conn and without pool" => fn ->
+           {:ok, %Tesla.Env{}} =
+             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+               adapter: [pool: :no_pool]
+             )
+         end,
+         "With reused conn and with pool" => fn ->
+           {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
+         end
+       },
+       parallel: 10
+     )
+   end
  end
index 42a311f99ce42c585030ce3698ebd52401cd92a8,23f94e3a609ada410da2636ec0fa51edf0ad2a90..e24f7923e455e3bb64939b17c9c5af9e00006844
@@@ -12,26 -12,6 +12,26 @@@ defmodule Pleroma.Web.MastodonAPI.Notif
  
    import Pleroma.Factory
  
 +  test "does NOT render account/pleroma/relationship if this is disabled by default" do
 +    clear_config([:extensions, :output_relationships_in_statuses_by_default], false)
 +
 +    %{user: user, conn: conn} = oauth_access(["read:notifications"])
 +    other_user = insert(:user)
 +
 +    {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"})
 +    {:ok, [_notification]} = Notification.create_notifications(activity)
 +
 +    response =
 +      conn
 +      |> assign(:user, user)
 +      |> get("/api/v1/notifications")
 +      |> json_response(200)
 +
 +    assert Enum.all?(response, fn n ->
 +             get_in(n, ["account", "pleroma", "relationship"]) == %{}
 +           end)
 +  end
 +
    test "list of notifications" do
      %{user: user, conn: conn} = oauth_access(["read:notifications"])
      other_user = insert(:user)
      assert length(json_response(conn, 200)) == 1
    end
  
+   @tag capture_log: true
    test "see move notifications" do
      old_user = insert(:user)
      new_user = insert(:user, also_known_as: [old_user.ap_id])
      %{user: follower, conn: conn} = oauth_access(["read:notifications"])
  
+     old_user_url = old_user.ap_id
+     body =
+       File.read!("test/fixtures/users_mock/localhost.json")
+       |> String.replace("{{nickname}}", old_user.nickname)
+       |> Jason.encode!()
+     Tesla.Mock.mock(fn
+       %{method: :get, url: ^old_user_url} ->
+         %Tesla.Env{status: 200, body: body}
+     end)
      User.follow(follower, old_user)
      Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
      Pleroma.Tests.ObanHelpers.perform_all()