Merge remote-tracking branch 'remotes/origin/develop' into feature/object-hashtags...
authorIvan Tashkinov <ivantashkinov@gmail.com>
Wed, 3 Feb 2021 06:31:38 +0000 (09:31 +0300)
committerIvan Tashkinov <ivantashkinov@gmail.com>
Wed, 3 Feb 2021 06:31:38 +0000 (09:31 +0300)
# Conflicts:
# CHANGELOG.md
# lib/pleroma/web/activity_pub/activity_pub.ex

1  2 
CHANGELOG.md
lib/pleroma/web/activity_pub/activity_pub.ex
lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex

diff --combined CHANGELOG.md
index 47c5078b8f953c222621c41249c164334f2d6375,3f439d3a44db8bbc571f04f35b438c024277d0b7..f9a32f430d6faaaf489f4435d5c3c00f322f84c0
@@@ -21,7 -21,7 +21,8 @@@ The format is based on [Keep a Changelo
  - Admin API: Reports now ordered by newest
  - Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders.
  - Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation verified with the included sample script
+ - Improve OAuth 2.0 provider support. A missing `fqn` field was added to the response, but does not expose the user's email address.
 +- Extracted object hashtags into separate table in order to improve hashtag timeline performance (via background migration in `Pleroma.Migrators.HashtagsTableMigrator`). 
  
  ### Added
  
  - Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
  - Admin API: An endpoint to manage frontends.
  - Streaming API: Add follow relationships updates.
- - WebPush: Introduce `pleroma:chat_mention` and `pleroma:emoji_reaction` notification types
+ - WebPush: Introduce `pleroma:chat_mention` and `pleroma:emoji_reaction` notification types.
+ - Mastodon API: Add monthly active users to `/api/v1/instance` (`pleroma.stats.mau`).
+ - Mastodon API: Home, public, hashtag & list timelines accept `only_media`, `remote` & `local` parameters for filtration.
+ - Mastodon API: `/api/v1/accounts/:id` & `/api/v1/mutes` endpoints accept `with_relationships` parameter and return filled `pleroma.relationship` field.
  </details>
  
  ### Fixed
@@@ -59,6 -62,9 +63,9 @@@
  - Creating incorrect IPv4 address-style HTTP links when encountering certain numbers.
  - Reblog API Endpoint: Do not set visibility parameter to public by default and let CommonAPI to infer it from status, so a user can reblog their private status without explicitly setting reblog visibility to private.
  - Tag URLs in statuses are now absolute
+ - Removed duplicate jobs to purge expired activities
+ - File extensions of some attachments were incorrectly changed. This feature has been disabled for now.
+ - Mix task pleroma.instance creates missing parent directories if the configuration or SQL output paths are changed.
  
  <details>
    <summary>API Changes</summary>
index 6cf4093fbada8dd6a34a46df3f2510915138c840,98051032aca1b10cb23353be274017556eed7449..573b4243c4d516e458dfe491847b4fab56e8cf0e
@@@ -669,140 -669,51 +669,140 @@@ defmodule Pleroma.Web.ActivityPub.Activ
  
    defp restrict_since(query, _), do: query
  
 -  defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
 -    raise "Can't use the child object without preloading!"
 +  defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
 +    raise_on_missing_preload()
 +  end
 +
 +  defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
 +    from(
 +      [_activity, object] in query,
 +      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
 +    )
 +  end
 +
 +  defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
 +    restrict_embedded_tag_any(query, %{tag: tag})
 +  end
 +
 +  defp restrict_embedded_tag_all(query, _), do: query
 +
 +  defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do
 +    raise_on_missing_preload()
 +  end
 +
 +  defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag}) do
 +    from(
 +      [_activity, object] in query,
 +      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
 +    )
 +  end
 +
 +  defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do
 +    restrict_embedded_tag_any(query, %{tag: [tag]})
    end
  
 -  defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
 +  defp restrict_embedded_tag_any(query, _), do: query
 +
 +  defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
 +    raise_on_missing_preload()
 +  end
 +
 +  defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do
      from(
        [_activity, object] in query,
        where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
      )
    end
  
 -  defp restrict_tag_reject(query, _), do: query
 +  defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject})
 +       when is_binary(tag_reject) do
 +    restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]})
 +  end
 +
 +  defp restrict_embedded_tag_reject_any(query, _), do: query
  
 -  defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
 -    raise "Can't use the child object without preloading!"
 +  defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
 +    raise_on_missing_preload()
    end
  
 -  defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
 +  defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do
      from(
        [_activity, object] in query,
 -      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
 +      where:
 +        fragment(
 +          """
 +          (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects
 +            ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?)
 +              AND hashtags_objects.object_id = ?) @> ?
 +          """,
 +          ^tags,
 +          object.id,
 +          ^tags
 +        )
      )
    end
  
 -  defp restrict_tag_all(query, _), do: query
 +  defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
 +    restrict_hashtag_any(query, %{tag: tag})
 +  end
  
 -  defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
 -    raise "Can't use the child object without preloading!"
 +  defp restrict_hashtag_all(query, _), do: query
 +
 +  defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
 +    raise_on_missing_preload()
    end
  
 -  defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
 +  defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do
      from(
        [_activity, object] in query,
 -      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
 +      where:
 +        fragment(
 +          """
 +          EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects
 +            ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?)
 +              AND hashtags_objects.object_id = ? LIMIT 1)
 +          """,
 +          ^tags,
 +          object.id
 +        )
      )
    end
  
 -  defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
 +  defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
 +    restrict_hashtag_any(query, %{tag: [tag]})
 +  end
 +
 +  defp restrict_hashtag_any(query, _), do: query
 +
 +  defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
 +    raise_on_missing_preload()
 +  end
 +
 +  defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do
      from(
        [_activity, object] in query,
 -      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
 +      where:
 +        fragment(
 +          """
 +          NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects
 +            ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?)
 +              AND hashtags_objects.object_id = ? LIMIT 1)
 +          """,
 +          ^tags_reject,
 +          object.id
 +        )
      )
    end
  
 -  defp restrict_tag(query, _), do: query
 +  defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
 +    restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
 +  end
 +
 +  defp restrict_hashtag_reject_any(query, _), do: query
 +
 +  defp raise_on_missing_preload do
 +    raise "Can't use the child object without preloading!"
 +  end
  
    defp restrict_recipients(query, [], _user), do: query
  
  
    defp restrict_local(query, _), do: query
  
+   defp restrict_remote(query, %{remote: true}) do
+     from(activity in query, where: activity.local == false)
+   end
+   defp restrict_remote(query, _), do: query
    defp restrict_actor(query, %{actor_id: actor_id}) do
      from(activity in query, where: activity.actor == ^actor_id)
    end
        skip_thread_containment: Config.get([:instance, :skip_thread_containment])
      }
  
 -    Activity
 -    |> maybe_preload_objects(opts)
 -    |> maybe_preload_bookmarks(opts)
 -    |> maybe_preload_report_notes(opts)
 -    |> maybe_set_thread_muted_field(opts)
 -    |> maybe_order(opts)
 -    |> restrict_recipients(recipients, opts[:user])
 -    |> restrict_replies(opts)
 -    |> restrict_tag(opts)
 -    |> restrict_tag_reject(opts)
 -    |> restrict_tag_all(opts)
 -    |> restrict_since(opts)
 -    |> restrict_local(opts)
 -    |> restrict_remote(opts)
 -    |> restrict_actor(opts)
 -    |> restrict_type(opts)
 -    |> restrict_state(opts)
 -    |> restrict_favorited_by(opts)
 -    |> restrict_blocked(restrict_blocked_opts)
 -    |> restrict_muted(restrict_muted_opts)
 -    |> restrict_filtered(opts)
 -    |> restrict_media(opts)
 -    |> restrict_visibility(opts)
 -    |> restrict_thread_visibility(opts, config)
 -    |> restrict_reblogs(opts)
 -    |> restrict_pinned(opts)
 -    |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
 -    |> restrict_instance(opts)
 -    |> restrict_announce_object_actor(opts)
 -    |> restrict_filtered(opts)
 -    |> Activity.restrict_deactivated_users()
 -    |> exclude_poll_votes(opts)
 -    |> exclude_chat_messages(opts)
 -    |> exclude_invisible_actors(opts)
 -    |> exclude_visibility(opts)
 +    query =
 +      Activity
 +      |> maybe_preload_objects(opts)
 +      |> maybe_preload_bookmarks(opts)
 +      |> maybe_preload_report_notes(opts)
 +      |> maybe_set_thread_muted_field(opts)
 +      |> maybe_order(opts)
 +      |> restrict_recipients(recipients, opts[:user])
 +      |> restrict_replies(opts)
 +      |> restrict_since(opts)
 +      |> restrict_local(opts)
++      |> restrict_remote(opts)
 +      |> restrict_actor(opts)
 +      |> restrict_type(opts)
 +      |> restrict_state(opts)
 +      |> restrict_favorited_by(opts)
 +      |> restrict_blocked(restrict_blocked_opts)
 +      |> restrict_muted(restrict_muted_opts)
 +      |> restrict_filtered(opts)
 +      |> restrict_media(opts)
 +      |> restrict_visibility(opts)
 +      |> restrict_thread_visibility(opts, config)
 +      |> restrict_reblogs(opts)
 +      |> restrict_pinned(opts)
 +      |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
 +      |> restrict_instance(opts)
 +      |> restrict_announce_object_actor(opts)
 +      |> restrict_filtered(opts)
 +      |> Activity.restrict_deactivated_users()
 +      |> exclude_poll_votes(opts)
 +      |> exclude_chat_messages(opts)
 +      |> exclude_invisible_actors(opts)
 +      |> exclude_visibility(opts)
 +
 +    if Config.improved_hashtag_timeline() do
 +      query
 +      |> restrict_hashtag_any(opts)
 +      |> restrict_hashtag_all(opts)
 +      |> restrict_hashtag_reject_any(opts)
 +    else
 +      query
 +      |> restrict_embedded_tag_any(opts)
 +      |> restrict_embedded_tag_all(opts)
 +      |> restrict_embedded_tag_reject_any(opts)
 +    end
    end
  
    def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
index 1fb954a9b1007f695eb13ee5116171fc26db5252,cef299aa49a1579719b3f017aade8edfced9076a..87effa00bcce3f431f93d4a4077a926c692e4192
@@@ -51,6 -51,8 +51,8 @@@ defmodule Pleroma.Web.MastodonAPI.Timel
        |> Map.put(:reply_filtering_user, user)
        |> Map.put(:announce_filtering_user, user)
        |> Map.put(:user, user)
+       |> Map.put(:local_only, params[:local])
+       |> Map.delete(:local)
  
      activities =
        [user.ap_id | User.following(user)]
      tags =
        [params[:tag], params[:any]]
        |> List.flatten()
 -      |> Enum.uniq()
        |> Enum.reject(&is_nil/1)
        |> Enum.map(&String.downcase/1)
 +      |> Enum.uniq()
  
      tag_all =
        params
          |> Map.put(:blocking_user, user)
          |> Map.put(:user, user)
          |> Map.put(:muting_user, user)
+         |> Map.put(:local_only, params[:local])
  
        # we must filter the following list for the user to avoid leaking statuses the user
        # does not actually have permission to see (for more info, peruse security issue #270).