X-Git-Url: https://git.squeep.com/?a=blobdiff_plain;ds=sidebyside;f=lib%2Fpleroma%2Fweb%2Factivity_pub%2Factivity_pub.ex;h=b74af3f3b5f57267a4376c06566a20aca9d0fcd5;hb=c9449326747f8d33357f5179e69d3024b39089a0;hp=339843330f8a3c96fa376b0d5580e0d6f6e26deb;hpb=303055456f19152821ec5ec1df88d60c03f60905;p=akkoma

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 339843330..b74af3f3b 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Conversation
   alias Pleroma.Conversation.Participation
   alias Pleroma.Filter
+  alias Pleroma.Hashtag
   alias Pleroma.Maps
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -33,6 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   require Pleroma.Constants
 
   @behaviour Pleroma.Web.ActivityPub.ActivityPub.Persisting
+  @behaviour Pleroma.Web.ActivityPub.ActivityPub.Streaming
 
   defp get_recipients(%{"type" => "Create"} = data) do
     to = Map.get(data, "to", [])
@@ -55,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp check_actor_is_active(actor) when is_binary(actor) do
     case User.get_cached_by_ap_id(actor) do
-      %User{deactivated: deactivated} -> not deactivated
+      %User{is_active: true} -> true
       _ -> false
     end
   end
@@ -86,7 +88,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
   @impl true
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
@@ -224,6 +226,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     Streamer.stream("participation", participations)
   end
 
+  @impl true
   def stream_out_participations(%Object{data: %{"context" => context}}, user) do
     with %Conversation{} = conversation <- Conversation.get_for_ap_id(context) do
       conversation = Repo.preload(conversation, :participations)
@@ -240,8 +243,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  @impl true
   def stream_out_participations(_, _), do: :noop
 
+  @impl true
   def stream_out(%Activity{data: %{"type" => data_type}} = activity)
       when data_type in ["Create", "Announce", "Delete"] do
     activity
@@ -249,6 +254,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Streamer.stream(activity)
   end
 
+  @impl true
   def stream_out(_activity) do
     :noop
   end
@@ -372,6 +378,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          :ok <-
            maybe_federate(stripped_activity) do
       User.all_superusers()
+      |> Enum.filter(fn user -> user.ap_id != actor end)
       |> Enum.filter(fn user -> not is_nil(user.email) end)
       |> Enum.each(fn superuser ->
         superuser
@@ -459,6 +466,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Repo.one()
   end
 
+  defp fetch_paginated_optimized(query, opts, pagination) do
+    # Note: tag-filtering funcs may apply "ORDER BY objects.id DESC",
+    #   and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan
+    opts = Map.put(opts, :skip_extra_order, true)
+
+    Pagination.fetch_paginated(query, opts, pagination)
+  end
+
+  def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
+    list_memberships = Pleroma.List.memberships(opts[:user])
+
+    fetch_activities_query(recipients ++ list_memberships, opts)
+    |> fetch_paginated_optimized(opts, pagination)
+    |> Enum.reverse()
+    |> maybe_update_cc(list_memberships, opts[:user])
+  end
+
   @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
   def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
     opts = Map.delete(opts, :user)
@@ -466,7 +490,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     [Constants.as_public()]
     |> fetch_activities_query(opts)
     |> restrict_unlisted(opts)
-    |> Pagination.fetch_paginated(opts, pagination)
+    |> fetch_paginated_optimized(opts, pagination)
   end
 
   @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
@@ -586,7 +610,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Enum.reverse()
   end
 
-  def fetch_user_activities(user, reading_user, params \\ %{}) do
+  def fetch_user_activities(user, reading_user, params \\ %{})
+
+  def fetch_user_activities(user, reading_user, %{total: true} = params) do
+    result = fetch_activities_for_user(user, reading_user, params)
+
+    Keyword.put(result, :items, Enum.reverse(result[:items]))
+  end
+
+  def fetch_user_activities(user, reading_user, params) do
+    user
+    |> fetch_activities_for_user(reading_user, params)
+    |> Enum.reverse()
+  end
+
+  defp fetch_activities_for_user(user, reading_user, params) do
     params =
       params
       |> Map.put(:type, ["Create", "Announce"])
@@ -603,16 +641,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         |> Map.put(:muting_user, reading_user)
       end
 
+    pagination_type = Map.get(params, :pagination_type) || :keyset
+
     %{
       godmode: params[:godmode],
       reading_user: reading_user
     }
     |> user_activities_recipients()
-    |> fetch_activities(params)
-    |> Enum.reverse()
+    |> fetch_activities(params, pagination_type)
+  end
+
+  def fetch_statuses(reading_user, %{total: true} = params) do
+    result = fetch_activities_for_reading_user(reading_user, params)
+    Keyword.put(result, :items, Enum.reverse(result[:items]))
   end
 
   def fetch_statuses(reading_user, params) do
+    reading_user
+    |> fetch_activities_for_reading_user(params)
+    |> Enum.reverse()
+  end
+
+  defp fetch_activities_for_reading_user(reading_user, params) do
     params = Map.put(params, :type, ["Create", "Announce"])
 
     %{
@@ -621,7 +671,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     }
     |> user_activities_recipients()
     |> fetch_activities(params, :offset)
-    |> Enum.reverse()
   end
 
   defp user_activities_recipients(%{godmode: true}), do: []
@@ -662,191 +711,139 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_since(query, _), do: query
 
-  defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+  defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
     raise_on_missing_preload()
   end
 
-  defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do
+  defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
     from(
       [_activity, object] in query,
-      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
+      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
     )
   end
 
-  defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
-    restrict_tag_reject(query, %{tag_reject: [tag_reject]})
+  defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
+    restrict_embedded_tag_any(query, %{tag: tag})
   end
 
-  defp restrict_tag_reject(query, _), do: query
+  defp restrict_embedded_tag_all(query, _), do: query
 
-  defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
+  defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do
     raise_on_missing_preload()
   end
 
-  defp restrict_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do
+  defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
+      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any)
     )
   end
 
-  defp restrict_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
-    restrict_tag(query, %{tag: tag})
+  defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do
+    restrict_embedded_tag_any(query, %{tag: [tag]})
   end
 
-  defp restrict_tag_all(query, _), do: query
+  defp restrict_embedded_tag_any(query, _), do: query
 
-  defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
+  defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
     raise_on_missing_preload()
   end
 
-  defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
+  defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
+      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
     )
   end
 
-  defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
-    restrict_tag(query, %{tag: [tag]})
+  defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject})
+       when is_binary(tag_reject) do
+    restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]})
   end
 
-  defp restrict_tag(query, _), do: query
-
-  defp restrict_hashtag(query, opts) do
-    [tag_any, tag_all, tag_reject] =
-      [:tag, :tag_all, :tag_reject]
-      |> Enum.map(&opts[&1])
-      |> Enum.map(&List.wrap(&1))
+  defp restrict_embedded_tag_reject_any(query, _), do: query
 
-    has_conditions = Enum.any?([tag_any, tag_all, tag_reject], &Enum.any?(&1))
-
-    cond do
-      !has_conditions ->
-        query
-
-      opts[:skip_preload] ->
-        raise_on_missing_preload()
-
-      true ->
-        query
-        |> group_by_all_bindings()
-        |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag)
-        |> maybe_restrict_hashtag_any(tag_any)
-        |> maybe_restrict_hashtag_all(tag_all)
-        |> maybe_restrict_hashtag_reject_any(tag_reject)
-    end
+  defp object_ids_query_for_tags(tags) do
+    from(hto in "hashtags_objects")
+    |> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id)
+    |> where([hto, ht], ht.name in ^tags)
+    |> select([hto], hto.object_id)
+    |> distinct([hto], true)
   end
 
-  # Groups by all bindings to allow aggregation on hashtags
-  defp group_by_all_bindings(query) do
-    # Expecting named bindings: :object, :bookmark, :thread_mute, :report_note
-    cond do
-      Enum.count(query.aliases) == 4 ->
-        from([a, o, b3, b4, b5] in query, group_by: [a.id, o.id, b3.id, b4.id, b5.id])
-
-      Enum.count(query.aliases) == 3 ->
-        from([a, o, b3, b4] in query, group_by: [a.id, o.id, b3.id, b4.id])
-
-      Enum.count(query.aliases) == 2 ->
-        from([a, o, b3] in query, group_by: [a.id, o.id, b3.id])
-
-      true ->
-        from([a, o] in query, group_by: [a.id, o.id])
-    end
-  end
-
-  defp maybe_restrict_hashtag_any(query, []) do
-    query
-  end
-
-  defp maybe_restrict_hashtag_any(query, tags) do
-    having(
-      query,
-      [hashtag: hashtag],
-      fragment("array_agg(?) && (?)", hashtag.name, ^tags)
-    )
+  defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
+    raise_on_missing_preload()
   end
 
-  defp maybe_restrict_hashtag_all(query, []) do
-    query
+  defp restrict_hashtag_all(query, %{tag_all: [single_tag]}) do
+    restrict_hashtag_any(query, %{tag: single_tag})
   end
 
-  defp maybe_restrict_hashtag_all(query, tags) do
-    having(
-      query,
-      [hashtag: hashtag],
-      fragment("array_agg(?) @> (?)", hashtag.name, ^tags)
+  defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do
+    from(
+      [_activity, object] in query,
+      where:
+        fragment(
+          """
+          (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects
+            ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?)
+              AND hashtags_objects.object_id = ?) @> ?
+          """,
+          ^tags,
+          object.id,
+          ^tags
+        )
     )
   end
 
-  defp maybe_restrict_hashtag_reject_any(query, []) do
-    query
+  defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
+    restrict_hashtag_all(query, %{tag_all: [tag]})
   end
 
-  defp maybe_restrict_hashtag_reject_any(query, tags) do
-    having(
-      query,
-      [hashtag: hashtag],
-      fragment("not(array_agg(?) && (?))", hashtag.name, ^tags)
-    )
-  end
+  defp restrict_hashtag_all(query, _), do: query
 
-  defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+  defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
     raise_on_missing_preload()
   end
 
-  defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do
-    query
-    |> group_by_all_bindings()
-    |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag)
-    |> having(
-      [hashtag: hashtag],
-      fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject)
-    )
-  end
-
-  defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
-    restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
-  end
-
-  defp restrict_hashtag_reject_any(query, _), do: query
-
-  defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
-    raise_on_missing_preload()
-  end
+  defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do
+    hashtag_ids =
+      from(ht in Hashtag, where: ht.name in ^tags, select: ht.id)
+      |> Repo.all()
 
-  defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do
-    Enum.reduce(
-      tags,
-      query,
-      fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end
+    # Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan
+    from(
+      [_activity, object] in query,
+      join: hto in "hashtags_objects",
+      on: hto.object_id == object.id,
+      where: hto.hashtag_id in ^hashtag_ids,
+      distinct: [desc: object.id],
+      order_by: [desc: object.id]
     )
   end
 
-  defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
-    restrict_hashtag_any(query, %{tag: tag})
+  defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
+    restrict_hashtag_any(query, %{tag: [tag]})
   end
 
-  defp restrict_hashtag_all(query, _), do: query
+  defp restrict_hashtag_any(query, _), do: query
 
-  defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
+  defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
     raise_on_missing_preload()
   end
 
-  defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do
+  defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do
     from(
       [_activity, object] in query,
-      join: hashtag in assoc(object, :hashtags),
-      where: hashtag.name in ^tags
+      where: object.id not in subquery(object_ids_query_for_tags(tags_reject))
     )
   end
 
-  defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
-    restrict_hashtag_any(query, %{tag: [tag]})
+  defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+    restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
   end
 
-  defp restrict_hashtag_any(query, _), do: query
+  defp restrict_hashtag_reject_any(query, _), do: query
 
   defp raise_on_missing_preload do
     raise "Can't use the child object without preloading!"
@@ -872,6 +869,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_local(query, _), do: query
 
+  defp restrict_remote(query, %{remote: true}) do
+    from(activity in query, where: activity.local == false)
+  end
+
+  defp restrict_remote(query, _), do: query
+
   defp restrict_actor(query, %{actor_id: actor_id}) do
     from(activity in query, where: activity.actor == ^actor_id)
   end
@@ -1205,6 +1208,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp maybe_order(query, _), do: query
 
+  defp normalize_fetch_activities_query_opts(opts) do
+    Enum.reduce([:tag, :tag_all, :tag_reject], opts, fn key, opts ->
+      case opts[key] do
+        value when is_bitstring(value) ->
+          Map.put(opts, key, Hashtag.normalize_name(value))
+
+        value when is_list(value) ->
+          normalized_value =
+            value
+            |> Enum.map(&Hashtag.normalize_name/1)
+            |> Enum.uniq()
+
+          Map.put(opts, key, normalized_value)
+
+        _ ->
+          opts
+      end
+    end)
+  end
+
   defp fetch_activities_query_ap_ids_ops(opts) do
     source_user = opts[:muting_user]
     ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
@@ -1228,6 +1251,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def fetch_activities_query(recipients, opts \\ %{}) do
+    opts = normalize_fetch_activities_query_opts(opts)
+
     {restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts} =
       fetch_activities_query_ap_ids_ops(opts)
 
@@ -1246,6 +1271,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> restrict_replies(opts)
       |> restrict_since(opts)
       |> restrict_local(opts)
+      |> restrict_remote(opts)
       |> restrict_actor(opts)
       |> restrict_type(opts)
       |> restrict_state(opts)
@@ -1268,35 +1294,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       |> exclude_invisible_actors(opts)
       |> exclude_visibility(opts)
 
-    cond do
-      Config.object_embedded_hashtags?() ->
-        query
-        |> restrict_tag(opts)
-        |> restrict_tag_reject(opts)
-        |> restrict_tag_all(opts)
-
-      # TODO: benchmark (initial approach preferring non-aggregate ops when possible)
-      Config.get([:instance, :improved_hashtag_timeline]) == :join ->
-        query
-        |> distinct([activity], true)
-        |> restrict_hashtag_any(opts)
-        |> restrict_hashtag_all(opts)
-        |> restrict_hashtag_reject_any(opts)
-
-      true ->
-        restrict_hashtag(query, opts)
+    if Config.feature_enabled?(:improved_hashtag_timeline) do
+      query
+      |> restrict_hashtag_any(opts)
+      |> restrict_hashtag_all(opts)
+      |> restrict_hashtag_reject_any(opts)
+    else
+      query
+      |> restrict_embedded_tag_any(opts)
+      |> restrict_embedded_tag_all(opts)
+      |> restrict_embedded_tag_reject_any(opts)
     end
   end
 
-  def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
-    list_memberships = Pleroma.List.memberships(opts[:user])
-
-    fetch_activities_query(recipients ++ list_memberships, opts)
-    |> Pagination.fetch_paginated(opts, pagination)
-    |> Enum.reverse()
-    |> maybe_update_cc(list_memberships, opts[:user])
-  end
-
   @doc """
   Fetch favorites activities of user with order by sort adds to favorites
   """
@@ -1373,21 +1383,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp get_actor_url(_url), do: nil
 
-  defp object_to_user_data(data) do
-    avatar =
-      data["icon"]["url"] &&
-        %{
-          "type" => "Image",
-          "url" => [%{"href" => data["icon"]["url"]}]
-        }
+  defp normalize_image(%{"url" => url}) do
+    %{
+      "type" => "Image",
+      "url" => [%{"href" => url}]
+    }
+  end
 
-    banner =
-      data["image"]["url"] &&
-        %{
-          "type" => "Image",
-          "url" => [%{"href" => data["image"]["url"]}]
-        }
+  defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
+  defp normalize_image(_), do: nil
 
+  defp object_to_user_data(data) do
     fields =
       data
       |> Map.get("attachment", [])
@@ -1431,13 +1437,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       ap_id: data["id"],
       uri: get_actor_url(data["url"]),
       ap_enabled: true,
-      banner: banner,
+      banner: normalize_image(data["image"]),
       fields: fields,
       emoji: emojis,
       is_locked: is_locked,
       is_discoverable: is_discoverable,
       invisible: invisible,
-      avatar: avatar,
+      avatar: normalize_image(data["icon"]),
       name: data["name"],
       follower_address: data["followers"],
       following_address: data["following"],