Merge branch 'develop' into feature/addressable-lists
authorEgor Kislitsyn <egor@kislitsyn.com>
Thu, 11 Jul 2019 06:26:59 +0000 (13:26 +0700)
committerEgor Kislitsyn <egor@kislitsyn.com>
Thu, 11 Jul 2019 06:26:59 +0000 (13:26 +0700)
14 files changed:
CHANGELOG.md
docs/api/differences_in_mastoapi_responses.md
lib/pleroma/list.ex
lib/pleroma/web/activity_pub/activity_pub.ex
lib/pleroma/web/activity_pub/publisher.ex
lib/pleroma/web/activity_pub/transmogrifier.ex
lib/pleroma/web/common_api/common_api.ex
lib/pleroma/web/common_api/utils.ex
lib/pleroma/web/salmon/salmon.ex
priv/repo/migrations/20190516112144_add_ap_id_to_lists.exs [new file with mode: 0644]
test/list_test.exs
test/web/activity_pub/activity_pub_test.exs
test/web/activity_pub/transmogrifier_test.exs
test/web/common_api/common_api_test.exs

index f27446f36b954a63cbb71d13781edeb4970c5c92..ff0cb8740ead2c989fdaff28f619fa71f9b1c34d 100644 (file)
@@ -89,6 +89,7 @@ Configuration: `federation_incoming_replies_max_depth` option
 - MRF: Support for rejecting reports from specific instances (`mrf_simple`)
 - MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`)
 - MRF: Support for running subchains.
+- Addressable lists
 - Configuration: `skip_thread_containment` option
 - Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details.
 - MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links.
index 2cbe1458dfe768dc5b799cc3c0c93a2a4944884d..1de1e3b44f262935534cad0d5b237f7e5e92d9be 100644 (file)
@@ -80,6 +80,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `preview`: boolean, if set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
 - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
+- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
 
 ## PATCH `/api/v1/update_credentials`
 
index a5b1cad680ddf20dc3eb23e5981917cfa765a840..16955b3b559bf3f34bd339941a1267be58a814bc 100644 (file)
@@ -16,6 +16,7 @@ defmodule Pleroma.List do
     belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
+    field(:ap_id, :string)
 
     timestamps()
   end
@@ -55,6 +56,10 @@ defmodule Pleroma.List do
     Repo.one(query)
   end
 
+  def get_by_ap_id(ap_id) do
+    Repo.get_by(__MODULE__, ap_id: ap_id)
+  end
+
   def get_following(%Pleroma.List{following: following} = _list) do
     q =
       from(
@@ -105,7 +110,14 @@ defmodule Pleroma.List do
 
   def create(title, %User{} = creator) do
     list = %Pleroma.List{user_id: creator.id, title: title}
-    Repo.insert(list)
+
+    Repo.transaction(fn ->
+      list = Repo.insert!(list)
+
+      list
+      |> change(ap_id: "#{creator.ap_id}/lists/#{list.id}")
+      |> Repo.update!()
+    end)
   end
 
   def follow(%Pleroma.List{following: following} = list, %User{} = followed) do
@@ -125,4 +137,13 @@ defmodule Pleroma.List do
     |> follow_changeset(attrs)
     |> Repo.update()
   end
+
+  def memberships(%User{follower_address: follower_address}) do
+    Pleroma.List
+    |> where([l], ^follower_address in l.following)
+    |> select([l], l.ap_id)
+    |> Repo.all()
+  end
+
+  def memberships(_), do: []
 end
index 41b55bbabcf1ddfd24674a49d13b6b97aaa3939b..20f72e676237a3e378bd15100858c7c579247848 100644 (file)
@@ -26,19 +26,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   # For Announce activities, we filter the recipients based on following status for any actors
   # that match actual users.  See issue #164 for more information about why this is necessary.
   defp get_recipients(%{"type" => "Announce"} = data) do
-    to = data["to"] || []
-    cc = data["cc"] || []
+    to = Map.get(data, "to", [])
+    cc = Map.get(data, "cc", [])
+    bcc = Map.get(data, "bcc", [])
     actor = User.get_cached_by_ap_id(data["actor"])
 
     recipients =
-      (to ++ cc)
-      |> Enum.filter(fn recipient ->
+      Enum.filter(Enum.concat([to, cc, bcc]), fn recipient ->
         case User.get_cached_by_ap_id(recipient) do
-          nil ->
-            true
-
-          user ->
-            User.following?(user, actor)
+          nil -> true
+          user -> User.following?(user, actor)
         end
       end)
 
@@ -46,17 +43,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp get_recipients(%{"type" => "Create"} = data) do
-    to = data["to"] || []
-    cc = data["cc"] || []
-    actor = data["actor"] || []
-    recipients = (to ++ cc ++ [actor]) |> Enum.uniq()
+    to = Map.get(data, "to", [])
+    cc = Map.get(data, "cc", [])
+    bcc = Map.get(data, "bcc", [])
+    actor = Map.get(data, "actor", [])
+    recipients = [to, cc, bcc, [actor]] |> Enum.concat() |> Enum.uniq()
     {recipients, to, cc}
   end
 
   defp get_recipients(data) do
-    to = data["to"] || []
-    cc = data["cc"] || []
-    recipients = to ++ cc
+    to = Map.get(data, "to", [])
+    cc = Map.get(data, "cc", [])
+    bcc = Map.get(data, "bcc", [])
+    recipients = Enum.concat([to, cc, bcc])
     {recipients, to, cc}
   end
 
@@ -896,13 +895,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp maybe_order(query, _), do: query
 
   def fetch_activities_query(recipients, opts \\ %{}) do
-    base_query = from(activity in Activity)
-
     config = %{
       skip_thread_containment: Config.get([:instance, :skip_thread_containment])
     }
 
-    base_query
+    Activity
     |> maybe_preload_objects(opts)
     |> maybe_preload_bookmarks(opts)
     |> maybe_set_thread_muted_field(opts)
@@ -931,11 +928,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def fetch_activities(recipients, opts \\ %{}) do
-    fetch_activities_query(recipients, opts)
+    list_memberships = Pleroma.List.memberships(opts["user"])
+
+    fetch_activities_query(recipients ++ list_memberships, opts)
     |> Pagination.fetch_paginated(opts)
     |> Enum.reverse()
+    |> maybe_update_cc(list_memberships, opts["user"])
   end
 
+  defp maybe_update_cc(activities, list_memberships, %User{ap_id: user_ap_id})
+       when is_list(list_memberships) and length(list_memberships) > 0 do
+    Enum.map(activities, fn
+      %{data: %{"bcc" => bcc}} = activity when is_list(bcc) and length(bcc) > 0 ->
+        if Enum.any?(bcc, &(&1 in list_memberships)) do
+          update_in(activity.data["cc"], &[user_ap_id | &1])
+        else
+          activity
+        end
+
+      activity ->
+        activity
+    end)
+  end
+
+  defp maybe_update_cc(activities, _, _), do: activities
+
   def fetch_activities_bounded_query(query, recipients, recipients_with_public) do
     from(activity in query,
       where:
index a05e032639ed542893f88eb1fafa6b7b48885028..b7dc90caa554e4a68896c66170a01fedfe3a0cc5 100644 (file)
@@ -92,18 +92,67 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     end
   end
 
-  @doc """
-  Publishes an activity to all relevant peers.
-  """
-  def publish(%User{} = actor, %Activity{} = activity) do
-    remote_followers =
+  defp recipients(actor, activity) do
+    followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
-        followers |> Enum.filter(&(!&1.local))
+        Enum.filter(followers, &(!&1.local))
       else
         []
       end
 
+    Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers
+  end
+
+  defp get_cc_ap_ids(ap_id, recipients) do
+    host = Map.get(URI.parse(ap_id), :host)
+
+    recipients
+    |> Enum.filter(fn %User{ap_id: ap_id} -> Map.get(URI.parse(ap_id), :host) == host end)
+    |> Enum.map(& &1.ap_id)
+  end
+
+  @doc """
+  Publishes an activity with BCC to all relevant peers.
+  """
+
+  def publish(actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do
+    public = is_public?(activity)
+    {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
+
+    recipients = recipients(actor, activity)
+
+    recipients
+    |> Enum.filter(&User.ap_enabled?/1)
+    |> Enum.map(fn %{info: %{source_data: data}} -> data["inbox"] end)
+    |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+    |> Instances.filter_reachable()
+    |> Enum.each(fn {inbox, unreachable_since} ->
+      %User{ap_id: ap_id} =
+        Enum.find(recipients, fn %{info: %{source_data: data}} -> data["inbox"] == inbox end)
+
+      cc = get_cc_ap_ids(ap_id, recipients)
+
+      json =
+        data
+        |> Map.put("cc", cc)
+        |> Map.put("directMessage", true)
+        |> Jason.encode!()
+
+      Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{
+        inbox: inbox,
+        json: json,
+        actor: actor,
+        id: activity.data["id"],
+        unreachable_since: unreachable_since
+      })
+    end)
+  end
+
+  @doc """
+  Publishes an activity to all relevant peers.
+  """
+  def publish(%User{} = actor, %Activity{} = activity) do
     public = is_public?(activity)
 
     if public && Config.get([:instance, :allow_relay]) do
@@ -114,7 +163,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
 
-    (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
+    recipients(actor, activity)
     |> Enum.filter(fn user -> User.ap_enabled?(user) end)
     |> Enum.map(fn %{info: %{source_data: data}} ->
       (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
index e34fe661100923c5a5f786078a1d194d76286bfe..ad741122ff59919f6863e0bd93421e1315a46a80 100644 (file)
@@ -814,13 +814,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do
     object =
-      Object.normalize(object_id).data
+      object_id
+      |> Object.normalize()
+      |> Map.get(:data)
       |> prepare_object
 
     data =
       data
       |> Map.put("object", object)
       |> Map.merge(Utils.make_json_ld_header())
+      |> Map.delete("bcc")
 
     {:ok, data}
   end
index f1450b1139178d940a78f854232613b67980ca9c..8e3892bdf778d3743c9219d34551b7b817d00f51 100644 (file)
@@ -175,6 +175,11 @@ defmodule Pleroma.Web.CommonAPI do
       when visibility in ~w{public unlisted private direct},
       do: {visibility, get_replied_to_visibility(in_reply_to)}
 
+  def get_visibility(%{"visibility" => "list:" <> list_id}, in_reply_to) do
+    visibility = {:list, String.to_integer(list_id)}
+    {visibility, get_replied_to_visibility(in_reply_to)}
+  end
+
   def get_visibility(_, in_reply_to) when not is_nil(in_reply_to) do
     visibility = get_replied_to_visibility(in_reply_to)
     {visibility, visibility}
@@ -210,6 +215,7 @@ defmodule Pleroma.Web.CommonAPI do
          addressed_users <- get_addressed_users(mentioned_users, data["to"]),
          {poll, poll_emoji} <- make_poll_data(data),
          {to, cc} <- get_to_and_cc(user, addressed_users, in_reply_to, visibility),
+         bcc <- bcc_for_list(user, visibility),
          context <- make_context(in_reply_to),
          cw <- data["spoiler_text"] || "",
          sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
@@ -235,19 +241,16 @@ defmodule Pleroma.Web.CommonAPI do
              "emoji",
              Map.merge(Formatter.get_emoji_map(full_payload), poll_emoji)
            ) do
-      res =
-        ActivityPub.create(
-          %{
-            to: to,
-            actor: user,
-            context: context,
-            object: object,
-            additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
-          },
-          Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
-        )
-
-      res
+      ActivityPub.create(
+        %{
+          to: to,
+          actor: user,
+          context: context,
+          object: object,
+          additional: %{"cc" => cc, "bcc" => bcc, "directMessage" => visibility == "direct"}
+        },
+        Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false
+      )
     else
       {:private_to_public, true} ->
         {:error, dgettext("errors", "The message visibility must be direct")}
index 8e482eef7b88f9e9520e40e70b039598e0ee3d01..d4bfdd7e40fd55cec353391f2295d01ed7ceab86 100644 (file)
@@ -100,12 +100,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end
   end
 
+  def get_to_and_cc(_user, _mentions, _inReplyTo, _), do: {[], []}
+
   def get_addressed_users(_, to) when is_list(to) do
     User.get_ap_ids_by_nicknames(to)
   end
 
   def get_addressed_users(mentioned_users, _), do: mentioned_users
 
+  def bcc_for_list(user, {:list, list_id}) do
+    list = Pleroma.List.get(list_id, user)
+    [list.ap_id]
+  end
+
+  def bcc_for_list(_, _), do: []
+
   def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_in}} = data)
       when is_list(options) do
     %{max_expiration: max_expiration, min_expiration: min_expiration} =
index e96e4e1e411e489ace59316c5fcebdb8d1e1ab4f..9b01ebcc642ea30acc7f09024bc5f7c9c0fea58b 100644 (file)
@@ -123,11 +123,26 @@ defmodule Pleroma.Web.Salmon do
     {:ok, salmon}
   end
 
-  def remote_users(%{data: %{"to" => to} = data}) do
-    to = to ++ (data["cc"] || [])
+  def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do
+    cc = Map.get(data, "cc", [])
+
+    bcc =
+      data
+      |> Map.get("bcc", [])
+      |> Enum.reduce([], fn ap_id, bcc ->
+        case Pleroma.List.get_by_ap_id(ap_id) do
+          %Pleroma.List{user_id: ^user_id} = list ->
+            {:ok, following} = Pleroma.List.get_following(list)
+            bcc ++ Enum.map(following, & &1.ap_id)
+
+          _ ->
+            bcc
+        end
+      end)
 
-    to
-    |> Enum.map(fn id -> User.get_cached_by_ap_id(id) end)
+    [to, cc, bcc]
+    |> Enum.concat()
+    |> Enum.map(&User.get_cached_by_ap_id/1)
     |> Enum.filter(fn user -> user && !user.local end)
   end
 
@@ -191,7 +206,7 @@ defmodule Pleroma.Web.Salmon do
       {:ok, private, _} = Keys.keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
-      remote_users = remote_users(activity)
+      remote_users = remote_users(user, activity)
 
       salmon_urls = Enum.map(remote_users, & &1.info.salmon)
       reachable_urls_metadata = Instances.filter_reachable(salmon_urls)
diff --git a/priv/repo/migrations/20190516112144_add_ap_id_to_lists.exs b/priv/repo/migrations/20190516112144_add_ap_id_to_lists.exs
new file mode 100644 (file)
index 0000000..3c32bc3
--- /dev/null
@@ -0,0 +1,26 @@
+defmodule Pleroma.Repo.Migrations.AddApIdToLists do
+  use Ecto.Migration
+
+  def up do
+    alter table(:lists) do
+      add(:ap_id, :string)
+    end
+
+    execute("""
+    UPDATE lists
+    SET ap_id = u.ap_id || '/lists/' || lists.id
+    FROM users AS u
+    WHERE lists.user_id = u.id
+    """)
+
+    create(unique_index(:lists, :ap_id))
+  end
+
+  def down do
+    drop(index(:lists, [:ap_id]))
+
+    alter table(:lists) do
+      remove(:ap_id)
+    end
+  end
+end
index 1909c0cd9e664771e7e425eab8542c5e7d8e8a2a..6c5c6b197f84e6ba514f2e8b6b9bb352163af349 100644 (file)
@@ -113,4 +113,19 @@ defmodule Pleroma.ListTest do
     assert owned_list in lists_2
     refute not_owned_list in lists_2
   end
+
+  test "get by ap_id" do
+    user = insert(:user)
+    {:ok, list} = Pleroma.List.create("foo", user)
+    assert Pleroma.List.get_by_ap_id(list.ap_id) == list
+  end
+
+  test "memberships" do
+    user = insert(:user)
+    member = insert(:user)
+    {:ok, list} = Pleroma.List.create("foo", user)
+    {:ok, list} = Pleroma.List.follow(list, member)
+
+    assert Pleroma.List.memberships(member) == [list.ap_id]
+  end
 end
index 59d56f3a758324633b0d70953ebd076f5117f3ca..00adbc0f9a7425d2b684b1fb7ed6a7fa755ae303 100644 (file)
@@ -1190,6 +1190,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     end
   end
 
+  test "fetch_activities/2 returns activities addressed to a list " do
+    user = insert(:user)
+    member = insert(:user)
+    {:ok, list} = Pleroma.List.create("foo", user)
+    {:ok, list} = Pleroma.List.follow(list, member)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
+
+    activity = Repo.preload(activity, :bookmark)
+    activity = %Activity{activity | thread_muted?: !!activity.thread_muted?}
+
+    assert ActivityPub.fetch_activities([], %{"user" => user}) == [activity]
+  end
+
   def data_uri do
     File.read!("test/fixtures/avatar_data_uri")
   end
index 825e998793cf6170ea99f37d5ea1c8806c3591b8..eb9300769d3547ba825fde5b52c3bc171160e588 100644 (file)
@@ -1096,6 +1096,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
 
       assert modified["directMessage"] == true
     end
+
+    test "it strips BCC field" do
+      user = insert(:user)
+      {:ok, list} = Pleroma.List.create("foo", user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
+
+      {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert is_nil(modified["bcc"])
+    end
   end
 
   describe "user upgrade" do
index 958c931c49a21353d6cdf5f9ac8a286ec1f3c1e0..694b52356d621b45f1161d2e30c9a0a9ccd80732 100644 (file)
@@ -129,6 +129,17 @@ defmodule Pleroma.Web.CommonAPITest do
                  })
       end)
     end
+
+    test "it allows to address a list" do
+      user = insert(:user)
+      {:ok, list} = Pleroma.List.create("foo", user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{"status" => "foobar", "visibility" => "list:#{list.id}"})
+
+      assert activity.data["bcc"] == [list.ap_id]
+      assert activity.recipients == [list.ap_id, user.ap_id]
+    end
   end
 
   describe "reactions" do