follow_handshake_timeout: 500,
note_replies_output_limit: 5,
sign_object_fetches: true,
- authorized_fetch_mode: false
+ authorized_fetch_mode: false,
+ max_collection_objects: 50
config :pleroma, :streamer,
workers: 3,
type: :integer,
description: "Following handshake timeout",
suggestions: [500]
+ },
+ %{
+ key: :max_collection_objects,
+ type: :integer,
+ description:
+ "The maximum number of items to fetch from a remote collections. Setting this too low can lead to only getting partial collections, but too high and you can end up fetching far too many objects.",
+ suggestions: [50]
}
]
},
* `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question
* `sign_object_fetches`: Sign object fetches with HTTP signatures
* `authorized_fetch_mode`: Require HTTP signatures for AP fetches
+* `max_collection_objects`: The maximum number of objects to fetch from a remote AP collection.
## Pleroma.User
--- /dev/null
+# Akkoma: The cooler fediverse server
+# Copyright © 2022- Akkoma Authors <https://akkoma.dev/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Akkoma.Collections.Fetcher do
+ @moduledoc """
+ Activitypub Collections fetching functions
+ see: https://www.w3.org/TR/activitystreams-core/#paging
+ """
+ alias Pleroma.Object.Fetcher
+ alias Pleroma.Config
+
+ def fetch_collection_by_ap_id(ap_id) when is_binary(ap_id) do
+ fetch_collection(ap_id)
+ end
+
+ def fetch_collection(ap_id) when is_binary(ap_id) do
+ with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
+ {:ok, objects_from_collection(page)}
+ else
+ e ->
+ Logger.error("Could not fetch collection #{ap_id} - #{inspect(e)}")
+ e
+ end
+ end
+
+ def fetch_collection(%{"type" => type} = page)
+ when type in ["Collection", "OrderedCollection"] do
+ {:ok, objects_from_collection(page)}
+ end
+
+ defp items_in_page(%{"type" => type, "orderedItems" => items})
+ when is_list(items) and type in ["OrderedCollection", "OrderedCollectionPage"],
+ do: items
+
+ defp items_in_page(%{"type" => type, "items" => items})
+ when is_list(items) and type in ["Collection", "CollectionPage"],
+ do: items
+
+ defp objects_from_collection(%{"type" => "OrderedCollection", "orderedItems" => items})
+ when is_list(items),
+ do: items
+
+ defp objects_from_collection(%{"type" => "Collection", "items" => items}) when is_list(items),
+ do: items
+
+ defp objects_from_collection(%{"type" => type, "first" => first})
+ when is_binary(first) and type in ["Collection", "OrderedCollection"] do
+ fetch_page_items(first)
+ end
+
+ defp objects_from_collection(%{"type" => type, "first" => %{"id" => id}})
+ when is_binary(id) and type in ["Collection", "OrderedCollection"] do
+ fetch_page_items(id)
+ end
+
+ defp fetch_page_items(id, items \\ []) do
+ if Enum.count(items) >= Config.get([:activitypub, :max_collection_objects]) do
+ items
+ else
+ {:ok, page} = Fetcher.fetch_and_contain_remote_object_from_id(id)
+ objects = items_in_page(page)
+
+ if Enum.count(objects) > 0 do
+ maybe_next_page(page, items ++ objects)
+ else
+ items
+ end
+ end
+ end
+
+ defp maybe_next_page(%{"next" => id}, items) when is_binary(id) do
+ fetch_page_items(id, items)
+ end
+
+ defp maybe_next_page(_, items), do: items
+end
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ActivityPub do
+ alias Akkoma.Collections
alias Pleroma.Activity
alias Pleroma.Activity.Ir.Topics
alias Pleroma.Config
end
def pin_data_from_featured_collection(%{
- "type" => type,
- "orderedItems" => objects
- })
+ "type" => "OrderedCollection",
+ "first" => first
+ }) do
+ with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(first) do
+ page
+ |> Map.get("orderedItems")
+ |> Map.new(fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end)
+ else
+ e ->
+ Logger.error("Could not decode featured collection at fetch #{first}, #{inspect(e)}")
+ {:ok, %{}}
+ end
+ end
+
+ def pin_data_from_featured_collection(
+ %{
+ "type" => type
+ } = collection
+ )
when type in ["OrderedCollection", "Collection"] do
+ {:ok, objects} = Collections.Fetcher.fetch_collection(collection)
Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end)
end
--- /dev/null
+{
+ "@context": "https://www.w3.org/ns/activitystreams",
+ "id": "https://example.com/collection/ordered_array",
+ "summary": "Object history",
+ "type": "OrderedCollection",
+ "totalItems": 2,
+ "orderedItems": [
+ {
+ "type": "Create",
+ "actor": "http://www.test.example/sally",
+ "object": "http://example.org/foo"
+ },
+ {
+ "type": "Like",
+ "actor": "http://www.test.example/joe",
+ "object": "http://example.org/foo"
+ }
+ ]
+}
--- /dev/null
+{
+ "@context": "https://www.w3.org/ns/activitystreams",
+ "id": "https://example.com/collection/unordered_array",
+ "summary": "Object history",
+ "type": "Collection",
+ "totalItems": 2,
+ "items": [
+ {
+ "type": "Create",
+ "actor": "http://www.test.example/sally",
+ "object": "http://example.org/foo"
+ },
+ {
+ "type": "Like",
+ "actor": "http://www.test.example/joe",
+ "object": "http://example.org/foo"
+ }
+ ]
+}
--- /dev/null
+{
+ "@context": "https://www.w3.org/ns/activitystreams",
+ "summary": "Sally's recent activities",
+ "type": "Collection",
+ "id": "http://example.org/foo",
+ "totalItems": 10,
+ "first": {
+ "type": "CollectionPage",
+ "id": "http://example.org/foo?page=1",
+ "partOf": "http://example.org/foo",
+ "next": "http://example.org/foo?page=2",
+ "items": [
+ {
+ "type": "Create",
+ "actor": "http://www.test.example/sally",
+ "object": "http://example.org/foo"
+ }
+ ]
+ }
+}
--- /dev/null
+{
+ "type": "CollectionPage",
+ "id": "https://example.com/collection/unordered_page_reference?page=1",
+ "partOf": "https://example.com/collection/unordered_page_reference",
+ "next": "https://example.com/collection/unordered_page_reference?page=2",
+ "items": [
+ {
+ "type": "Create",
+ "actor": "http://www.test.example/sally",
+ "object": "http://example.org/foo"
+ }
+ ]
+}
--- /dev/null
+{
+ "@context": "https://www.w3.org/ns/activitystreams",
+ "summary": "Sally's recent activities",
+ "type": "Collection",
+ "id": "https://example.com/collection/unordered_page_reference",
+ "totalItems": 10,
+ "first": "https://example.com/collection/unordered_page_reference?page=1"
+}
--- /dev/null
+{
+ "type": "CollectionPage",
+ "id": "https://example.com/collection/unordered_page_reference?page=2",
+ "partOf": "https://example.com/collection/unordered_page_reference",
+ "items": [
+ {
+ "type": "Like",
+ "actor": "http://www.test.example/sally",
+ "object": "http://example.org/foo"
+ }
+ ]
+}
--- /dev/null
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ "https://w3id.org/security/v1",
+ {
+ "vcard": "http://www.w3.org/2006/vcard/ns#",
+ "dfrn": "http://purl.org/macgirvin/dfrn/1.0/",
+ "diaspora": "https://diasporafoundation.org/ns/",
+ "litepub": "http://litepub.social/ns#",
+ "toot": "http://joinmastodon.org/ns#",
+ "featured": {
+ "@id": "toot:featured",
+ "@type": "@id"
+ },
+ "schema": "http://schema.org#",
+ "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+ "sensitive": "as:sensitive",
+ "Hashtag": "as:Hashtag",
+ "directMessage": "litepub:directMessage",
+ "discoverable": "toot:discoverable",
+ "PropertyValue": "schema:PropertyValue",
+ "value": "schema:value"
+ }
+ ],
+ "id": "https://friendica.example.com/featured/raha",
+ "type": "OrderedCollection",
+ "totalItems": 0,
+ "first": "https://friendica.example.com/featured/raha?page=1"
+}
--- /dev/null
+{
+ "@context": [
+ "https://www.w3.org/ns/activitystreams",
+ "https://w3id.org/security/v1",
+ {
+ "vcard": "http://www.w3.org/2006/vcard/ns#",
+ "dfrn": "http://purl.org/macgirvin/dfrn/1.0/",
+ "diaspora": "https://diasporafoundation.org/ns/",
+ "litepub": "http://litepub.social/ns#",
+ "toot": "http://joinmastodon.org/ns#",
+ "featured": {
+ "@id": "toot:featured",
+ "@type": "@id"
+ },
+ "schema": "http://schema.org#",
+ "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+ "sensitive": "as:sensitive",
+ "Hashtag": "as:Hashtag",
+ "directMessage": "litepub:directMessage",
+ "discoverable": "toot:discoverable",
+ "PropertyValue": "schema:PropertyValue",
+ "value": "schema:value"
+ }
+ ],
+ "id": "https://friendica.example.com/featured/raha?page=1",
+ "type": "OrderedCollectionPage",
+ "totalItems": 0,
+ "partOf": "https://friendica.example.com/featured/raha",
+ "orderedItems": [
+ {
+ "id": "http://inserted"
+ }
+ ]
+}
--- /dev/null
+defmodule Akkoma.Collections.FetcherTest do
+ use Pleroma.DataCase
+ use Oban.Testing, repo: Pleroma.Repo
+
+ alias Akkoma.Collections.Fetcher
+
+ import Tesla.Mock
+
+ setup do
+ mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+ :ok
+ end
+
+ test "it should extract items from an embedded array in a Collection" do
+ unordered_collection =
+ "test/fixtures/collections/unordered_array.json"
+ |> File.read!()
+
+ ap_id = "https://example.com/collection/ordered_array"
+
+ Tesla.Mock.mock(fn
+ %{
+ method: :get,
+ url: ^ap_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: unordered_collection,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+ end)
+
+ {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
+ assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
+ end
+
+ test "it should extract items from an embedded array in an OrderedCollection" do
+ ordered_collection =
+ "test/fixtures/collections/ordered_array.json"
+ |> File.read!()
+
+ ap_id = "https://example.com/collection/ordered_array"
+
+ Tesla.Mock.mock(fn
+ %{
+ method: :get,
+ url: ^ap_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: ordered_collection,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+ end)
+
+ {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
+ assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
+ end
+
+ test "it should extract items from an referenced first page in a Collection" do
+ unordered_collection =
+ "test/fixtures/collections/unordered_page_reference.json"
+ |> File.read!()
+
+ first_page =
+ "test/fixtures/collections/unordered_page_first.json"
+ |> File.read!()
+
+ second_page =
+ "test/fixtures/collections/unordered_page_second.json"
+ |> File.read!()
+
+ ap_id = "https://example.com/collection/unordered_page_reference"
+ first_page_id = "https://example.com/collection/unordered_page_reference?page=1"
+ second_page_id = "https://example.com/collection/unordered_page_reference?page=2"
+
+ Tesla.Mock.mock(fn
+ %{
+ method: :get,
+ url: ^ap_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: unordered_collection,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+
+ %{
+ method: :get,
+ url: ^first_page_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: first_page,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+
+ %{
+ method: :get,
+ url: ^second_page_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: second_page,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+ end)
+
+ {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
+ assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects
+ end
+
+ test "it should stop fetching when we hit :max_collection_objects" do
+ clear_config([:activitypub, :max_collection_objects], 1)
+
+ unordered_collection =
+ "test/fixtures/collections/unordered_page_reference.json"
+ |> File.read!()
+
+ first_page =
+ "test/fixtures/collections/unordered_page_first.json"
+ |> File.read!()
+
+ second_page =
+ "test/fixtures/collections/unordered_page_second.json"
+ |> File.read!()
+
+ ap_id = "https://example.com/collection/unordered_page_reference"
+ first_page_id = "https://example.com/collection/unordered_page_reference?page=1"
+ second_page_id = "https://example.com/collection/unordered_page_reference?page=2"
+
+ Tesla.Mock.mock(fn
+ %{
+ method: :get,
+ url: ^ap_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: unordered_collection,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+
+ %{
+ method: :get,
+ url: ^first_page_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: first_page,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+
+ %{
+ method: :get,
+ url: ^second_page_id
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: second_page,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+ end)
+
+ {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id)
+ assert [%{"type" => "Create"}] = objects
+ end
+end
end
end
+ test "fetches user featured collection using the first property" do
+ featured_url = "https://friendica.example.com/raha/collections/featured"
+ first_url = "https://friendica.example.com/featured/raha?page=1"
+
+ featured_data =
+ "test/fixtures/friendica/friendica_featured_collection.json"
+ |> File.read!()
+
+ page_data =
+ "test/fixtures/friendica/friendica_featured_collection_first.json"
+ |> File.read!()
+
+ Tesla.Mock.mock(fn
+ %{
+ method: :get,
+ url: ^featured_url
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: featured_data,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+
+ %{
+ method: :get,
+ url: ^first_url
+ } ->
+ %Tesla.Env{
+ status: 200,
+ body: page_data,
+ headers: [{"content-type", "application/activity+json"}]
+ }
+ end)
+
+ {:ok, data} = ActivityPub.fetch_and_prepare_featured_from_ap_id(featured_url)
+ assert Map.has_key?(data, "http://inserted")
+ end
+
test "it fetches the appropriate tag-restricted posts" do
user = insert(:user)
projects / akkoma / commitdiff