From: kaniini Date: Fri, 4 Oct 2019 17:33:18 +0000 (+0000) Subject: Merge branch 'user-info-unread-direct-conversation' into 'develop' X-Git-Url: https://git.squeep.com/?a=commitdiff_plain;h=5c04e8172407660662d1e052e40a665bdb063ae9;hp=06d9df79c5c99069dd12e863c99167eb20b6495b;p=akkoma Merge branch 'user-info-unread-direct-conversation' into 'develop' Add the `unread_conversation_count` field to the user info See merge request pleroma/pleroma!1737 --- diff --git a/CHANGELOG.md b/CHANGELOG.md index a38f61fba..4b4048109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added - Refreshing poll results for remote polls +- Job queue stats to the healthcheck page - Admin API: Add ability to require password reset - Mastodon API: Account entities now include `follow_requests_count` (planned Mastodon 3.x addition) - Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items @@ -24,6 +25,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) +- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname` +- Added `:instance, extended_nickname_format` setting to the default config ## [1.1.0] - 2019-??-?? ### Security diff --git a/config/config.exs b/config/config.exs index 36bea19a0..ddbfb246a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -279,7 +279,8 @@ config :pleroma, :instance, max_remote_account_fields: 20, account_field_name_length: 512, account_field_value_length: 2048, - external_user_synchronization: true + external_user_synchronization: true, + extended_nickname_format: false config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 3a8ef4e2c..0517bbdd7 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -317,7 +317,8 @@ See [Admin-API](admin_api.md) "active": 0, # active processes "idle": 0, # idle processes "memory_used": 0.00, # Memory used - "healthy": true # Instance state + "healthy": true, # Instance state + "job_queue_stats": {} # Job queue stats } ``` @@ -391,7 +392,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ### Update a file in a custom emoji pack * Method `POST` * Authentication: required -* Params: +* Params: * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`, that means that the emoji file needs to be uploaded with the request (thus requiring it to be a multipart request) and be named `file`. @@ -408,7 +409,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ### Updates (replaces) pack metadata * Method `POST` * Authentication: required -* Params: +* Params: * `new_data`: new metadata to replace the old one * Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a problem with the new metadata (the error is specified in the "error" part of the response JSON) @@ -417,7 +418,7 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa ### Requests the instance to download the pack from another instance * Method `POST` * Authentication: required -* Params: +* Params: * `instance_address`: the address of the instance to download from * `pack_name`: the pack to download from that instance * Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 8f00915a3..b86799ecc 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -189,7 +189,7 @@ See the [Quack Github](https://github.com/azohra/quack) for more details ## :frontend_configurations -This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. +This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. You can find the documentation for `pleroma_fe` configuration into [Pleroma-FE configuration and customization for instance administrators](/frontend/CONFIGURATION/#options). Frontends can access these settings at `/api/pleroma/frontend_configurations` diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9e35b02c0..0bf218bc7 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,7 @@ defmodule Pleroma.Application do hackney_pool_children() ++ [ Pleroma.Stats, + Pleroma.JobQueueMonitor, {Oban, Pleroma.Config.get(Oban)} ] ++ task_children(@env) ++ diff --git a/lib/pleroma/healthcheck.ex b/lib/pleroma/healthcheck.ex index 977b78c26..fc2129815 100644 --- a/lib/pleroma/healthcheck.ex +++ b/lib/pleroma/healthcheck.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Healthcheck do active: 0, idle: 0, memory_used: 0, + job_queue_stats: nil, healthy: true @type t :: %__MODULE__{ @@ -21,6 +22,7 @@ defmodule Pleroma.Healthcheck do active: non_neg_integer(), idle: non_neg_integer(), memory_used: number(), + job_queue_stats: map(), healthy: boolean() } @@ -30,6 +32,7 @@ defmodule Pleroma.Healthcheck do memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2) } |> assign_db_info() + |> assign_job_queue_stats() |> check_health() end @@ -55,6 +58,11 @@ defmodule Pleroma.Healthcheck do Map.merge(healthcheck, db_info) end + defp assign_job_queue_stats(healthcheck) do + stats = Pleroma.JobQueueMonitor.stats() + Map.put(healthcheck, :job_queue_stats, stats) + end + @spec check_health(Healthcheck.t()) :: Healthcheck.t() def check_health(%{pool_size: pool_size, active: active} = check) when active >= pool_size do diff --git a/lib/pleroma/job_queue_monitor.ex b/lib/pleroma/job_queue_monitor.ex new file mode 100644 index 000000000..3feea8381 --- /dev/null +++ b/lib/pleroma/job_queue_monitor.ex @@ -0,0 +1,78 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobQueueMonitor do + use GenServer + + @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0} + @queue %{processed_jobs: 0, success: 0, failure: 0} + @operation %{processed_jobs: 0, success: 0, failure: 0} + + def start_link(_) do + GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__) + end + + @impl true + def init(state) do + :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil) + :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil) + + {:ok, state} + end + + def stats do + GenServer.call(__MODULE__, :stats) + end + + def handle_event([:oban, status], %{duration: duration}, meta, _) do + GenServer.cast(__MODULE__, {:process_event, status, duration, meta}) + end + + @impl true + def handle_call(:stats, _from, state) do + {:reply, state, state} + end + + @impl true + def handle_cast({:process_event, status, duration, meta}, state) do + state = + state + |> Map.update!(:workers, fn workers -> + workers + |> Map.put_new(meta.worker, %{}) + |> Map.update!(meta.worker, &update_worker(&1, status, meta, duration)) + end) + |> Map.update!(:queues, fn workers -> + workers + |> Map.put_new(meta.queue, @queue) + |> Map.update!(meta.queue, &update_queue(&1, status, meta, duration)) + end) + |> Map.update!(:processed_jobs, &(&1 + 1)) + + {:noreply, state} + end + + defp update_worker(worker, status, meta, duration) do + worker + |> Map.put_new(meta.args["op"], @operation) + |> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration)) + end + + defp update_op(op, :enqueue, _meta, _duration) do + op + |> Map.update!(:enqueued, &(&1 + 1)) + end + + defp update_op(op, status, _meta, _duration) do + op + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> Map.update!(status, &(&1 + 1)) + end + + defp update_queue(queue, status, _meta, _duration) do + queue + |> Map.update!(:processed_jobs, &(&1 + 1)) + |> Map.update!(status, &(&1 + 1)) + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 572dd7746..494a67f22 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -584,7 +584,7 @@ defmodule Pleroma.User do is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) -> get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id) - restrict_to_local == false -> + restrict_to_local == false or not String.contains?(nickname_or_id, "@") -> get_cached_by_nickname(nickname_or_id) restrict_to_local == :unauthenticated and match?(%User{}, opts[:for]) -> diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 0828591ee..ac5550671 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -461,14 +461,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do """ def make_unannounce_data( %User{ap_id: ap_id} = user, - %Activity{data: %{"context" => context}} = activity, + %Activity{data: %{"context" => context, "object" => object}} = activity, activity_id ) do + object = Object.normalize(object) + %{ "type" => "Undo", "actor" => ap_id, "object" => activity.data, - "to" => [user.follower_address, activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => context } @@ -477,14 +479,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do def make_unlike_data( %User{ap_id: ap_id} = user, - %Activity{data: %{"context" => context}} = activity, + %Activity{data: %{"context" => context, "object" => object}} = activity, activity_id ) do + object = Object.normalize(object) + %{ "type" => "Undo", "actor" => ap_id, "object" => activity.data, - "to" => [user.follower_address, activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => context } diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 60b58dc90..5e3dbe728 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -25,40 +25,44 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"]) mastodon_type = Activity.mastodon_notification_type(activity) - response = %{ - id: to_string(notification.id), - type: mastodon_type, - created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), - account: AccountView.render("show.json", %{user: actor, for: user}), - pleroma: %{ - is_seen: notification.seen + with %{id: _} = account <- AccountView.render("show.json", %{user: actor, for: user}) do + response = %{ + id: to_string(notification.id), + type: mastodon_type, + created_at: CommonAPI.Utils.to_masto_date(notification.inserted_at), + account: account, + pleroma: %{ + is_seen: notification.seen + } } - } - case mastodon_type do - "mention" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: activity, for: user}) - }) + case mastodon_type do + "mention" -> + response + |> Map.merge(%{ + status: StatusView.render("show.json", %{activity: activity, for: user}) + }) - "favourite" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: parent_activity, for: user}) - }) + "favourite" -> + response + |> Map.merge(%{ + status: StatusView.render("show.json", %{activity: parent_activity, for: user}) + }) - "reblog" -> - response - |> Map.merge(%{ - status: StatusView.render("show.json", %{activity: parent_activity, for: user}) - }) + "reblog" -> + response + |> Map.merge(%{ + status: StatusView.render("show.json", %{activity: parent_activity, for: user}) + }) - "follow" -> - response + "follow" -> + response - _ -> - nil + _ -> + nil + end + else + _ -> nil end end end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index e418dc70d..1cd7294e7 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -460,7 +460,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do end # Special case: Local MastodonFE - defp redirect_uri(%Plug.Conn{} = conn, "."), do: mastodon_api_url(conn, :login) + defp redirect_uri(%Plug.Conn{} = conn, "."), do: auth_url(conn, :login) defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri diff --git a/test/healthcheck_test.exs b/test/healthcheck_test.exs index 6bb8d5b7f..66d5026ff 100644 --- a/test/healthcheck_test.exs +++ b/test/healthcheck_test.exs @@ -9,7 +9,14 @@ defmodule Pleroma.HealthcheckTest do test "system_info/0" do result = Healthcheck.system_info() |> Map.from_struct() - assert Map.keys(result) == [:active, :healthy, :idle, :memory_used, :pool_size] + assert Map.keys(result) == [ + :active, + :healthy, + :idle, + :job_queue_stats, + :memory_used, + :pool_size + ] end describe "check_health/1" do diff --git a/test/job_queue_monitor_test.exs b/test/job_queue_monitor_test.exs new file mode 100644 index 000000000..17c6f3246 --- /dev/null +++ b/test/job_queue_monitor_test.exs @@ -0,0 +1,70 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.JobQueueMonitorTest do + use ExUnit.Case, async: true + + alias Pleroma.JobQueueMonitor + + @success {:process_event, :success, 1337, + %{ + args: %{"op" => "refresh_subscriptions"}, + attempt: 1, + id: 339, + max_attempts: 5, + queue: "federator_outgoing", + worker: "Pleroma.Workers.SubscriberWorker" + }} + + @failure {:process_event, :failure, 22_521_134, + %{ + args: %{"op" => "force_password_reset", "user_id" => "9nJG6n6Nbu7tj9GJX6"}, + attempt: 1, + error: %RuntimeError{message: "oops"}, + id: 345, + kind: :exception, + max_attempts: 1, + queue: "background", + stack: [ + {Pleroma.Workers.BackgroundWorker, :perform, 2, + [file: 'lib/pleroma/workers/background_worker.ex', line: 31]}, + {Oban.Queue.Executor, :safe_call, 1, + [file: 'lib/oban/queue/executor.ex', line: 42]}, + {:timer, :tc, 3, [file: 'timer.erl', line: 197]}, + {Oban.Queue.Executor, :call, 2, [file: 'lib/oban/queue/executor.ex', line: 23]}, + {Task.Supervised, :invoke_mfa, 2, [file: 'lib/task/supervised.ex', line: 90]}, + {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]} + ], + worker: "Pleroma.Workers.BackgroundWorker" + }} + + test "stats/0" do + assert %{processed_jobs: _, queues: _, workers: _} = JobQueueMonitor.stats() + end + + test "handle_cast/2" do + state = %{workers: %{}, queues: %{}, processed_jobs: 0} + + assert {:noreply, state} = JobQueueMonitor.handle_cast(@success, state) + assert {:noreply, state} = JobQueueMonitor.handle_cast(@failure, state) + assert {:noreply, state} = JobQueueMonitor.handle_cast(@success, state) + assert {:noreply, state} = JobQueueMonitor.handle_cast(@failure, state) + + assert state == %{ + processed_jobs: 4, + queues: %{ + "background" => %{failure: 2, processed_jobs: 2, success: 0}, + "federator_outgoing" => %{failure: 0, processed_jobs: 2, success: 2} + }, + workers: %{ + "Pleroma.Workers.BackgroundWorker" => %{ + "force_password_reset" => %{failure: 2, processed_jobs: 2, success: 0} + }, + "Pleroma.Workers.SubscriberWorker" => %{ + "refresh_subscriptions" => %{failure: 0, processed_jobs: 2, success: 2} + } + } + } + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 126bd69e8..1bc853c94 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1725,4 +1725,61 @@ defmodule Pleroma.UserTest do assert %{info: %{hide_follows: true}} = Repo.get(User, user.id) assert {:ok, %{info: %{hide_follows: true}}} = Cachex.get(:user_cache, "ap_id:#{user.ap_id}") end + + describe "get_cached_by_nickname_or_id" do + setup do + limit_to_local_content = Pleroma.Config.get([:instance, :limit_to_local_content]) + local_user = insert(:user) + remote_user = insert(:user, nickname: "nickname@example.com", local: false) + + on_exit(fn -> + Pleroma.Config.put([:instance, :limit_to_local_content], limit_to_local_content) + end) + + [local_user: local_user, remote_user: remote_user] + end + + test "allows getting remote users by id no matter what :limit_to_local_content is set to", %{ + remote_user: remote_user + } do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.id) + + Pleroma.Config.put([:instance, :limit_to_local_content], true) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.id) + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.id) + end + + test "disallows getting remote users by nickname without authentication when :limit_to_local_content is set to :unauthenticated", + %{remote_user: remote_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert nil == User.get_cached_by_nickname_or_id(remote_user.nickname) + end + + test "allows getting remote users by nickname with authentication when :limit_to_local_content is set to :unauthenticated", + %{remote_user: remote_user, local_user: local_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert %User{} = User.get_cached_by_nickname_or_id(remote_user.nickname, for: local_user) + end + + test "disallows getting remote users by nickname when :limit_to_local_content is set to true", + %{remote_user: remote_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], true) + assert nil == User.get_cached_by_nickname_or_id(remote_user.nickname) + end + + test "allows getting local users by nickname no matter what :limit_to_local_content is set to", + %{local_user: local_user} do + Pleroma.Config.put([:instance, :limit_to_local_content], false) + assert %User{} = User.get_cached_by_nickname_or_id(local_user.nickname) + + Pleroma.Config.put([:instance, :limit_to_local_content], true) + assert %User{} = User.get_cached_by_nickname_or_id(local_user.nickname) + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + assert %User{} = User.get_cached_by_nickname_or_id(local_user.nickname) + end + end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index f29497847..c9f2a92e7 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -811,10 +811,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, like_activity, object} = ActivityPub.like(user, object) assert object.data["like_count"] == 1 - {:ok, _, _, object} = ActivityPub.unlike(user, object) + {:ok, unlike_activity, _, object} = ActivityPub.unlike(user, object) assert object.data["like_count"] == 0 assert Activity.get_by_id(like_activity.id) == nil + assert note_activity.actor in unlike_activity.recipients end end @@ -890,7 +891,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do assert unannounce_activity.data["to"] == [ User.ap_followers(user), - announce_activity.data["actor"] + object.data["actor"] ] assert unannounce_activity.data["type"] == "Undo" diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index b1c1d6f71..c57ea7eb9 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -106,11 +106,13 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do user = insert(:user) like_activity = insert(:like_activity, data_attrs: %{"context" => "test context"}) + object = Object.normalize(like_activity.data["object"]) + assert Utils.make_unlike_data(user, like_activity, nil) == %{ "type" => "Undo", "actor" => user.ap_id, "object" => like_activity.data, - "to" => [user.follower_address, like_activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => like_activity.data["context"] } @@ -119,7 +121,7 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do "type" => "Undo", "actor" => user.ap_id, "object" => like_activity.data, - "to" => [user.follower_address, like_activity.data["actor"]], + "to" => [user.follower_address, object.data["actor"]], "cc" => [Pleroma.Constants.as_public()], "context" => like_activity.data["context"], "id" => "9mJEZK0tky1w2xD2vY" diff --git a/test/web/mastodon_api/views/notification_view_test.exs b/test/web/mastodon_api/views/notification_view_test.exs index 81ab82e2b..c9043a69a 100644 --- a/test/web/mastodon_api/views/notification_view_test.exs +++ b/test/web/mastodon_api/views/notification_view_test.exs @@ -100,5 +100,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationViewTest do NotificationView.render("index.json", %{notifications: [notification], for: followed}) assert [expected] == result + + User.perform(:delete, follower) + notification = Notification |> Repo.one() |> Repo.preload(:activity) + + assert [] == + NotificationView.render("index.json", %{notifications: [notification], for: followed}) end end