Merge branch 'features/validators-video' into 'develop'
authorrinpatch <rinpatch@sdf.org>
Wed, 16 Sep 2020 13:36:27 +0000 (13:36 +0000)
committerrinpatch <rinpatch@sdf.org>
Wed, 16 Sep 2020 13:36:27 +0000 (13:36 +0000)
Pipeline Ingestion: Video and Article

See merge request pleroma/pleroma!2908

36 files changed:
CHANGELOG.md
config/config.exs
config/description.exs
docs/configuration/cheatsheet.md
lib/mix/tasks/pleroma/database.ex
lib/mix/tasks/pleroma/user.ex
lib/pleroma/gun/conn.ex
lib/pleroma/gun/connection_pool/worker.ex
lib/pleroma/instances/instance.ex
lib/pleroma/migration_helper/notification_backfill.ex
lib/pleroma/repo.ex
lib/pleroma/repo_streamer.ex [deleted file]
lib/pleroma/telemetry/logger.ex
lib/pleroma/user.ex
lib/pleroma/web/activity_pub/activity_pub.ex
lib/pleroma/web/activity_pub/mrf/simple_policy.ex
lib/pleroma/web/activity_pub/transmogrifier.ex
lib/pleroma/web/rich_media/helpers.ex
lib/pleroma/web/rich_media/parser.ex
priv/gettext/es/LC_MESSAGES/errors.po
priv/repo/migrations/20200825061316_move_activity_expirations_to_oban.exs
priv/repo/migrations/20200907092050_move_tokens_expiration_into_oban.exs
priv/repo/migrations/20200910113106_remove_managed_config_from_db.exs [new file with mode: 0644]
priv/repo/migrations/20200911055909_remove_cron_jobs.exs [new file with mode: 0644]
priv/repo/migrations/20200914105638_delete_notification_without_activity.exs [new file with mode: 0644]
priv/repo/migrations/20200914105800_add_notification_constraints.exs [new file with mode: 0644]
test/marker_test.exs
test/repo_test.exs
test/support/http_request_mock.ex
test/user_test.exs
test/web/activity_pub/activity_pub_test.exs
test/web/instances/instance_test.exs
test/web/mastodon_api/controllers/account_controller_test.exs
test/web/mastodon_api/controllers/marker_controller_test.exs
test/web/mastodon_api/views/account_view_test.exs
test/web/rich_media/parser_test.exs

index 75357f05edc20cf20cc62c471e1bc6e85fb687ac..f7a372e1110dc3587054fc52d71fb598d9cf681e 100644 (file)
@@ -9,15 +9,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
+- Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 
 ### Removed
 
 - **Breaking:** `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab` (moved to a simpler implementation).
 - **Breaking:** `Pleroma.Workers.Cron.ClearOauthTokenWorker` setting from Oban `:crontab` (moved to scheduled jobs).
 - **Breaking:** `Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker` setting from Oban `:crontab` (moved to scheduled jobs).
+- Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
+switched to a new configuration mechanism, however it was not officially removed until now.
 
-### Changed
-- Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
+## unreleased-patch - ???
+
+### Fixed
+
+- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance
+- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user
 
 ## [2.1.1] - 2020-09-08
 
index 88c47fd032c382b9e92a021dcccdd4c5bbdba869..c204814d0573d0172e156b1ecb1c56b23977e47f 100644 (file)
@@ -216,7 +216,6 @@ config :pleroma, :instance,
   allow_relay: true,
   public: true,
   quarantined_instances: [],
-  managed_config: true,
   static_dir: "instance/static/",
   allowed_post_formats: [
     "text/plain",
index 82c7bc6a7d0c98a45b43d8323d120ed490c1320e..2b30f81489b5dd25769636ed2842da1486a668d1 100644 (file)
@@ -764,12 +764,6 @@ config :pleroma, :config_description, [
           "*.quarantined.com"
         ]
       },
-      %{
-        key: :managed_config,
-        type: :boolean,
-        description:
-          "Whenether the config for pleroma-fe is configured in this config or in static/config.json"
-      },
       %{
         key: :static_dir,
         type: :string,
index 7cf1d1ce789594c7e8e35f8df095d2b3b4618128..054b8fe4369d7a0b95cfea3fee067b220bde528a 100644 (file)
@@ -18,7 +18,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `notify_email`: Email used for notifications.
 * `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance``.
 * `limit`: Posts character limit (CW/Subject included in the counter).
-* `discription_limit`: The character limit for image descriptions.
+* `description_limit`: The character limit for image descriptions.
 * `chat_limit`: Character limit of the instance chat messages.
 * `remote_limit`: Hard character limit beyond which remote posts will be dropped.
 * `upload_limit`: File size limit of uploads (except for avatar, background, banner).
@@ -40,7 +40,6 @@ To add configuration to your config file, you can copy it from the base config.
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance.
 * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
 * `quarantined_instances`: List of ActivityPub instances where private (DMs, followers-only) activities will not be send.
-* `managed_config`: Whenether the config for pleroma-fe is configured in [:frontend_configurations](#frontend_configurations) or in ``static/config.json``.
 * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
index 7f1108dcfa87cee488266a2cc1343be7824ece75..a01c36ece30754ada443c22dffc5e3d29b9f73cf 100644 (file)
@@ -99,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       where: fragment("(?)->>'likes' is not null", object.data),
       select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
     )
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn objects ->
       ids =
         objects
@@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> where(local: true)
     |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
     |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity ->
         expires_at =
index 01824aa18b0f45dda65716d6189856f3394f63c1..b20c49d89538db746214e7b6b17256995d3879c8 100644 (file)
@@ -179,7 +179,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -370,7 +370,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{local: true})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
index 75b1ffc0a6a1cef762cc44de0815047eb5efb0d3..477e19c6e1d7589eccb1a9d1394451502d1571c5 100644 (file)
@@ -50,10 +50,10 @@ defmodule Pleroma.Gun.Conn do
 
     with open_opts <- Map.delete(opts, :tls_opts),
          {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
          stream <- Gun.connect(conn, connect_opts),
          {:response, :fin, 200, _} <- Gun.await(conn, stream) do
-      {:ok, conn}
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
@@ -88,8 +88,8 @@ defmodule Pleroma.Gun.Conn do
       |> Map.put(:socks_opts, socks_opts)
 
     with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
-      {:ok, conn}
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
@@ -106,8 +106,8 @@ defmodule Pleroma.Gun.Conn do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
 
     with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
-      {:ok, conn}
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
index c36332817d1c585a0f4776468f104bae639ffe05..49d41e4c7e62931b347469ca076c93566289b458 100644 (file)
@@ -15,7 +15,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
   @impl true
   def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
-    with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
+    with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
       time = :erlang.monotonic_time(:millisecond)
 
@@ -27,8 +27,12 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
       send(client_pid, {:conn_pid, conn_pid})
 
       {:noreply,
-       %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
-       :hibernate}
+       %{
+         key: key,
+         timer: nil,
+         client_monitors: %{client_pid => Process.monitor(client_pid)},
+         protocol: protocol
+       }, :hibernate}
     else
       err ->
         {:stop, {:shutdown, err}, nil}
@@ -53,14 +57,20 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   end
 
   @impl true
-  def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do
+  def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
     time = :erlang.monotonic_time(:millisecond)
 
-    {{conn_pid, _, _, _}, _} =
+    {{conn_pid, used_by, _, _}, _} =
       Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
         {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
       end)
 
+    :telemetry.execute(
+      [:pleroma, :connection_pool, :client, :add],
+      %{client_pid: client_pid, clients: used_by},
+      %{key: state.key, protocol: protocol}
+    )
+
     state =
       if state.timer != nil do
         Process.cancel_timer(state[:timer])
@@ -131,7 +141,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   @impl true
   def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
     :telemetry.execute(
-      [:pleroma, :connection_pool, :client_death],
+      [:pleroma, :connection_pool, :client, :dead],
       %{client_pid: pid, reason: reason},
       %{key: state.key}
     )
index 8bf53c090a8cc07a7df4ddb775af4cb8121535a3..6948651c7bacdbba4768fe8df50a68904a5eeae5 100644 (file)
@@ -159,13 +159,11 @@ defmodule Pleroma.Instances.Instance do
              Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
                adapter: [pool: :media]
              ),
-           favicon_rel <-
-             html
-             |> Floki.parse_document!()
-             |> Floki.attribute("link[rel=icon]", "href")
-             |> List.first(),
-           favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
-           true <- is_binary(favicon) do
+           {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
+             {:parse,
+              html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
+           {_, favicon} when is_binary(favicon) <-
+             {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
         favicon
       else
         _ -> nil
index d260e62cacde990436e314027b3d308619013354..24f4733fe796c47905c14f014bb5ff36a2cd21a2 100644 (file)
@@ -19,13 +19,13 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
     query
     |> Repo.chunk_stream(100)
     |> Enum.each(fn notification ->
-      type =
-        notification.activity
-        |> type_from_activity()
+      if notification.activity do
+        type = type_from_activity(notification.activity)
 
-      notification
-      |> Ecto.Changeset.change(%{type: type})
-      |> Repo.update()
+        notification
+        |> Ecto.Changeset.change(%{type: type})
+        |> Repo.update()
+      end
     end)
   end
 
@@ -72,8 +72,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
         "pleroma:emoji_reaction"
 
       "Create" ->
-        activity
-        |> type_from_activity_object()
+        type_from_activity_object(activity)
 
       t ->
         raise "No notification type for activity type #{t}"
index f317e4d582e6cea6fee974190e30015797cffb8a..4524bd5e2c766636382ba2c0c34d409d10c289b4 100644 (file)
@@ -49,7 +49,21 @@ defmodule Pleroma.Repo do
     end
   end
 
-  def chunk_stream(query, chunk_size) do
+  @doc """
+  Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+  `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+  ## Examples
+
+  # fetch records one-by-one
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+  # fetch records in bulk
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+  """
+  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+  def chunk_stream(query, chunk_size, returns_as \\ :one) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
     # have individual records be the elements of the stream, instead of
@@ -69,7 +83,12 @@ defmodule Pleroma.Repo do
 
             records ->
               last_id = List.last(records).id
-              {records, last_id}
+
+              if returns_as == :one do
+                {records, last_id}
+              else
+                {[records], last_id}
+              end
           end
       end,
       fn _ -> :ok end
diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex
deleted file mode 100644 (file)
index cb4d7bb..0000000
+++ /dev/null
@@ -1,34 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.RepoStreamer do
-  alias Pleroma.Repo
-  import Ecto.Query
-
-  def chunk_stream(query, chunk_size) do
-    Stream.unfold(0, fn
-      :halt ->
-        {[], :halt}
-
-      last_id ->
-        query
-        |> order_by(asc: :id)
-        |> where([r], r.id > ^last_id)
-        |> limit(^chunk_size)
-        |> Repo.all()
-        |> case do
-          [] ->
-            {[], :halt}
-
-          records ->
-            last_id = List.last(records).id
-            {records, last_id}
-        end
-    end)
-    |> Stream.take_while(fn
-      [] -> false
-      _ -> true
-    end)
-  end
-end
index 4cacae02f9bd010cea475f03dd50b6b2fc4d5ecc..197b1d091679fd85884f7ec40a7a3294e5333b9e 100644 (file)
@@ -7,7 +7,8 @@ defmodule Pleroma.Telemetry.Logger do
     [:pleroma, :connection_pool, :reclaim, :start],
     [:pleroma, :connection_pool, :reclaim, :stop],
     [:pleroma, :connection_pool, :provision_failure],
-    [:pleroma, :connection_pool, :client_death]
+    [:pleroma, :connection_pool, :client, :dead],
+    [:pleroma, :connection_pool, :client, :add]
   ]
   def attach do
     :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, [])
@@ -62,7 +63,7 @@ defmodule Pleroma.Telemetry.Logger do
   end
 
   def handle_event(
-        [:pleroma, :connection_pool, :client_death],
+        [:pleroma, :connection_pool, :client, :dead],
         %{client_pid: client_pid, reason: reason},
         %{key: key},
         _
@@ -73,4 +74,17 @@ defmodule Pleroma.Telemetry.Logger do
       }"
     end)
   end
+
+  def handle_event(
+        [:pleroma, :connection_pool, :client, :add],
+        %{clients: [_, _ | _] = clients},
+        %{key: key, protocol: :http},
+        _
+      ) do
+    Logger.info(fn ->
+      "Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur."
+    end)
+  end
+
+  def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok
 end
index e73d199648f616d0a57a1c9ad2014cc83fc77ac9..57497eb83e66da2f9a799346703df4c7d997f11d 100644 (file)
@@ -25,7 +25,6 @@ defmodule Pleroma.User do
   alias Pleroma.Object
   alias Pleroma.Registration
   alias Pleroma.Repo
-  alias Pleroma.RepoStreamer
   alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web
@@ -1775,7 +1774,7 @@ defmodule Pleroma.User do
   def delete_user_activities(%User{ap_id: ap_id} = user) do
     ap_id
     |> Activity.Queries.by_actor()
-    |> RepoStreamer.chunk_stream(50)
+    |> Repo.chunk_stream(50, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity -> delete_activity(activity, user) end)
     end)
index 3ab0457375a333a554ebe644abf458372ec7c872..f5cbb3e01922ad91568fd940a220dc0112ad1525 100644 (file)
@@ -767,7 +767,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_replies(query, %{
-         reply_filtering_user: user,
+         reply_filtering_user: %User{} = user,
          reply_visibility: "self"
        }) do
     from(
@@ -783,7 +783,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_replies(query, %{
-         reply_filtering_user: user,
+         reply_filtering_user: %User{} = user,
          reply_visibility: "following"
        }) do
     from(
index bb193475ab12a47f0aa6967584368b36b3373f33..16117772717091489ae3c7b5fa0b76e651bad7ab 100644 (file)
@@ -66,7 +66,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
            "type" => "Create",
            "object" => child_object
          } = object
-       ) do
+       )
+       when is_map(child_object) do
     media_nsfw =
       Config.get([:mrf_simple, :media_nsfw])
       |> MRF.subdomains_regex()
index 80f529704e31ea888d10db8af1e16d38225ac85e..fcca014f01d548f0302e2f0f08bcb2d938219cce 100644 (file)
@@ -290,7 +290,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do
     emoji =
       tags
-      |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end)
+      |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end)
       |> Enum.reduce(%{}, fn data, mapping ->
         name = String.trim(data["name"], ":")
 
index bd7f03cbeba8c3bf55bf86341e47440ae17464d9..d7a19df4a125e8b565106a02395ebc2b54cd8098 100644 (file)
@@ -87,6 +87,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
-    Pleroma.HTTP.get(url, headers, @options)
+    head_check =
+      case Pleroma.HTTP.head(url, headers, @options) do
+        # If the HEAD request didn't reach the server for whatever reason,
+        # we assume the GET that comes right after won't either
+        {:error, _} = e ->
+          e
+
+        {:ok, %Tesla.Env{status: 200, headers: headers}} ->
+          with :ok <- check_content_type(headers),
+               :ok <- check_content_length(headers),
+               do: :ok
+
+        _ ->
+          :ok
+      end
+
+    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options)
+  end
+
+  defp check_content_type(headers) do
+    case List.keyfind(headers, "content-type", 0) do
+      {_, content_type} ->
+        case Plug.Conn.Utils.media_type(content_type) do
+          {:ok, "text", "html", _} -> :ok
+          _ -> {:error, {:content_type, content_type}}
+        end
+
+      _ ->
+        :ok
+    end
+  end
+
+  @max_body @options[:max_body]
+  defp check_content_length(headers) do
+    case List.keyfind(headers, "content-length", 0) do
+      {_, maybe_content_length} ->
+        case Integer.parse(maybe_content_length) do
+          {content_length, ""} when content_length <= @max_body -> :ok
+          {_, ""} -> {:error, :body_too_large}
+          _ -> :ok
+        end
+
+      _ ->
+        :ok
+    end
   end
 end
index 5727fda189bf39f4ac501461504ff247f1f86fa7..33f6f1fa1bba2aab9ce02476af39499747066c92 100644 (file)
@@ -36,6 +36,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
         {:ok, _data} = res ->
           res
 
+        {:error, :body_too_large} = e ->
+          e
+
+        {:error, {:content_type, _}} = e ->
+          e
+
+        # The TTL is not set for the errors above, since they are unlikely to change
+        # with time
         {:error, _} = e ->
           ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
           Cachex.expire(:rich_media_cache, url, ttl)
index ba75936a96459ee5fe020601766fd799d4ef3c17..0a6fceaad0bbcc55374e2965272e0afd114691e1 100644 (file)
@@ -3,7 +3,7 @@ msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2020-09-09 09:49+0000\n"
-"PO-Revision-Date: 2020-09-09 10:52+0000\n"
+"PO-Revision-Date: 2020-09-11 21:26+0000\n"
 "Last-Translator: tarteka <info@tarteka.net>\n"
 "Language-Team: Spanish <https://translate.pleroma.social/projects/pleroma/"
 "pleroma/es/>\n"
@@ -94,52 +94,52 @@ msgid "must be less than %{number}"
 msgstr ""
 
 msgid "must be greater than %{number}"
-msgstr ""
+msgstr "debe ser mayor que %{number}"
 
 msgid "must be less than or equal to %{number}"
-msgstr ""
+msgstr "debe ser menor o igual que %{number}"
 
 msgid "must be greater than or equal to %{number}"
-msgstr ""
+msgstr "deber ser mayor o igual que %{number}"
 
 msgid "must be equal to %{number}"
-msgstr ""
+msgstr "deber ser igual a %{number}"
 
 #: lib/pleroma/web/common_api/common_api.ex:505
 #, elixir-format
 msgid "Account not found"
-msgstr ""
+msgstr "Cuenta no encontrada"
 
 #: lib/pleroma/web/common_api/common_api.ex:339
 #, elixir-format
 msgid "Already voted"
-msgstr ""
+msgstr "Ya has votado"
 
 #: lib/pleroma/web/oauth/oauth_controller.ex:359
 #, elixir-format
 msgid "Bad request"
-msgstr ""
+msgstr "Solicitud incorrecta"
 
 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:426
 #, elixir-format
 msgid "Can't delete object"
-msgstr ""
+msgstr "No se puede eliminar el objeto"
 
 #: lib/pleroma/web/controller_helper.ex:105
 #: lib/pleroma/web/controller_helper.ex:111
 #, elixir-format
 msgid "Can't display this activity"
-msgstr ""
+msgstr "No se puede mostrar esta actividad"
 
 #: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:285
 #, elixir-format
 msgid "Can't find user"
-msgstr ""
+msgstr "No se puede encontrar al usuario"
 
 #: lib/pleroma/web/pleroma_api/controllers/account_controller.ex:61
 #, elixir-format
 msgid "Can't get favorites"
-msgstr ""
+msgstr "No se puede obtener los favoritos"
 
 #: lib/pleroma/web/activity_pub/activity_pub_controller.ex:438
 #, elixir-format
@@ -149,7 +149,7 @@ msgstr ""
 #: lib/pleroma/web/common_api/utils.ex:563
 #, elixir-format
 msgid "Cannot post an empty status without attachments"
-msgstr ""
+msgstr "No se puede publicar un estado vacío y sin archivos adjuntos"
 
 #: lib/pleroma/web/common_api/utils.ex:511
 #, elixir-format
index cdc00d20baf97a5ea0bec858d08a73f770caea54..a703af83fd270064001df1554bb45153dc35d96b 100644 (file)
@@ -4,6 +4,8 @@ defmodule Pleroma.Repo.Migrations.MoveActivityExpirationsToOban do
   import Ecto.Query, only: [from: 2]
 
   def change do
+    Pleroma.Config.Oban.warn()
+
     Supervisor.start_link([{Oban, Pleroma.Config.get(Oban)}],
       strategy: :one_for_one,
       name: Pleroma.Supervisor
index 832bd02a75fe14d9d842eb1fffdbce5d2a3c1952..9e49ddacbb7fdc6802dc873229cdff1499f8aeeb 100644 (file)
@@ -4,6 +4,8 @@ defmodule Pleroma.Repo.Migrations.MoveTokensExpirationIntoOban do
   import Ecto.Query, only: [from: 2]
 
   def change do
+    Pleroma.Config.Oban.warn()
+
     Supervisor.start_link([{Oban, Pleroma.Config.get(Oban)}],
       strategy: :one_for_one,
       name: Pleroma.Supervisor
diff --git a/priv/repo/migrations/20200910113106_remove_managed_config_from_db.exs b/priv/repo/migrations/20200910113106_remove_managed_config_from_db.exs
new file mode 100644 (file)
index 0000000..e27a9ae
--- /dev/null
@@ -0,0 +1,27 @@
+defmodule Pleroma.Repo.Migrations.RemoveManagedConfigFromDb do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.ConfigDB
+  alias Pleroma.Repo
+
+  def up do
+    config_entry =
+      from(c in ConfigDB,
+        select: [:id, :value],
+        where: c.group == ^:pleroma and c.key == ^:instance
+      )
+      |> Repo.one()
+
+    if config_entry do
+      {_, value} = Keyword.pop(config_entry.value, :managed_config)
+
+      config_entry
+      |> Ecto.Changeset.change(value: value)
+      |> Repo.update()
+    end
+  end
+
+  def down do
+    :ok
+  end
+end
diff --git a/priv/repo/migrations/20200911055909_remove_cron_jobs.exs b/priv/repo/migrations/20200911055909_remove_cron_jobs.exs
new file mode 100644 (file)
index 0000000..33897d1
--- /dev/null
@@ -0,0 +1,20 @@
+defmodule Pleroma.Repo.Migrations.RemoveCronJobs do
+  use Ecto.Migration
+
+  import Ecto.Query, only: [from: 2]
+
+  def up do
+    from(j in "oban_jobs",
+      where:
+        j.worker in ^[
+          "Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker",
+          "Pleroma.Workers.Cron.StatsWorker",
+          "Pleroma.Workers.Cron.ClearOauthTokenWorker"
+        ],
+      select: [:id]
+    )
+    |> Pleroma.Repo.delete_all()
+  end
+
+  def down, do: :ok
+end
diff --git a/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs b/priv/repo/migrations/20200914105638_delete_notification_without_activity.exs
new file mode 100644 (file)
index 0000000..9333fc5
--- /dev/null
@@ -0,0 +1,30 @@
+defmodule Pleroma.Repo.Migrations.DeleteNotificationWithoutActivity do
+  use Ecto.Migration
+
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  def up do
+    from(
+      q in Pleroma.Notification,
+      left_join: c in assoc(q, :activity),
+      select: %{id: type(q.id, :integer)},
+      where: is_nil(c.id)
+    )
+    |> Repo.chunk_stream(1_000, :batches)
+    |> Stream.each(fn records ->
+      notification_ids = Enum.map(records, fn %{id: id} -> id end)
+
+      Repo.delete_all(
+        from(n in "notifications",
+          where: n.id in ^notification_ids
+        )
+      )
+    end)
+    |> Stream.run()
+  end
+
+  def down do
+    :ok
+  end
+end
diff --git a/priv/repo/migrations/20200914105800_add_notification_constraints.exs b/priv/repo/migrations/20200914105800_add_notification_constraints.exs
new file mode 100644 (file)
index 0000000..a65c35f
--- /dev/null
@@ -0,0 +1,23 @@
+defmodule Pleroma.Repo.Migrations.AddNotificationConstraints do
+  use Ecto.Migration
+
+  def up do
+    drop(constraint(:notifications, "notifications_activity_id_fkey"))
+
+    alter table(:notifications) do
+      modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all),
+        null: false
+      )
+    end
+  end
+
+  def down do
+    drop(constraint(:notifications, "notifications_activity_id_fkey"))
+
+    alter table(:notifications) do
+      modify(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all),
+        null: true
+      )
+    end
+  end
+end
index 5b6d0b4a4729b4af0886fefb25f013222a7eefe7..7b3943c7be5ca907b3c990daa46c72abc7d2a104 100644 (file)
@@ -33,8 +33,8 @@ defmodule Pleroma.MarkerTest do
     test "returns user markers" do
       user = insert(:user)
       marker = insert(:marker, user: user)
-      insert(:notification, user: user)
-      insert(:notification, user: user)
+      insert(:notification, user: user, activity: insert(:note_activity))
+      insert(:notification, user: user, activity: insert(:note_activity))
       insert(:marker, timeline: "home", user: user)
 
       assert Marker.get_markers(
index 92e827c950fb68401b38d2327f8fdc00d963fe85..155791be22fd1fb2905ca0c82d97d9d1f14d4f01 100644 (file)
@@ -37,7 +37,9 @@ defmodule Pleroma.RepoTest do
 
     test "get one-to-many assoc from repo" do
       user = insert(:user)
-      notification = refresh_record(insert(:notification, user: user))
+
+      notification =
+        refresh_record(insert(:notification, user: user, activity: insert(:note_activity)))
 
       assert Repo.get_assoc(user, :notifications) == {:ok, [notification]}
     end
@@ -47,4 +49,32 @@ defmodule Pleroma.RepoTest do
       assert Repo.get_assoc(token, :user) == {:error, :not_found}
     end
   end
+
+  describe "chunk_stream/3" do
+    test "fetch records one-by-one" do
+      users = insert_list(50, :user)
+
+      {fetch_users, 50} =
+        from(t in User)
+        |> Repo.chunk_stream(5)
+        |> Enum.reduce({[], 0}, fn %User{} = user, {acc, count} ->
+          {acc ++ [user], count + 1}
+        end)
+
+      assert users == fetch_users
+    end
+
+    test "fetch records in bulk" do
+      users = insert_list(50, :user)
+
+      {fetch_users, 10} =
+        from(t in User)
+        |> Repo.chunk_stream(5, :batches)
+        |> Enum.reduce({[], 0}, fn users, {acc, count} ->
+          {acc ++ users, count + 1}
+        end)
+
+      assert users == fetch_users
+    end
+  end
 end
index 344e27f137506a8fb65f43789b5df98a65c243ca..cb022333f369a718a3eaf2f4f8c4768eaacbdfb0 100644 (file)
@@ -1262,4 +1262,21 @@ defmodule HttpRequestMock do
        inspect(headers)
      }"}
   end
+
+  # Most of the rich media mocks are missing HEAD requests, so we just return 404.
+  @rich_media_mocks [
+    "https://example.com/ogp",
+    "https://example.com/ogp-missing-data",
+    "https://example.com/twitter-card"
+  ]
+  def head(url, _query, _body, _headers) when url in @rich_media_mocks do
+    {:ok, %Tesla.Env{status: 404, body: ""}}
+  end
+
+  def head(url, query, body, headers) do
+    {:error,
+     "Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{
+       inspect(headers)
+     }"}
+  end
 end
index 50f72549eeab1a61a683d479b27f20f185f3ed3a..a910226b2210ec9f32fa5dc4a0a35a61518ff2f8 100644 (file)
@@ -440,6 +440,45 @@ defmodule Pleroma.UserTest do
       assert activity.actor == welcome_user.ap_id
     end
 
+    setup do:
+            clear_config(:mrf_simple,
+              media_removal: [],
+              media_nsfw: [],
+              federated_timeline_removal: [],
+              report_removal: [],
+              reject: [],
+              followers_only: [],
+              accept: [],
+              avatar_removal: [],
+              banner_removal: [],
+              reject_deletes: []
+            )
+
+    setup do:
+            clear_config(:mrf,
+              policies: [
+                Pleroma.Web.ActivityPub.MRF.SimplePolicy
+              ]
+            )
+
+    test "it sends a welcome chat message when Simple policy applied to local instance" do
+      Pleroma.Config.put([:mrf_simple, :media_nsfw], ["localhost"])
+
+      welcome_user = insert(:user)
+      Pleroma.Config.put([:welcome, :chat_message, :enabled], true)
+      Pleroma.Config.put([:welcome, :chat_message, :sender_nickname], welcome_user.nickname)
+      Pleroma.Config.put([:welcome, :chat_message, :message], "Hello, this is a chat message")
+
+      cng = User.register_changeset(%User{}, @full_user_data)
+      {:ok, registered_user} = User.register(cng)
+      ObanHelpers.perform_all()
+
+      activity = Repo.one(Pleroma.Activity)
+      assert registered_user.ap_id in activity.recipients
+      assert Object.normalize(activity).data["content"] =~ "chat message"
+      assert activity.actor == welcome_user.ap_id
+    end
+
     test "it sends a welcome email message if it is set" do
       welcome_user = insert(:user)
       Pleroma.Config.put([:welcome, :email, :enabled], true)
index d8caa0b00b0d6162b38426f5fd93bd695a98e448..7bdad3810d960df6f8337d0d75d02a5835276203 100644 (file)
@@ -1810,6 +1810,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
         |> Enum.map(& &1.id)
 
       assert activities_ids == []
+
+      activities_ids =
+        %{}
+        |> Map.put(:reply_visibility, "self")
+        |> Map.put(:reply_filtering_user, nil)
+        |> ActivityPub.fetch_public_activities()
+
+      assert activities_ids == []
     end
 
     test "home timeline", %{users: %{u1: user}} do
index dc6ace843f43a69d475bc991601f4649024b6112..4f080510018d22ad827af469af29f791bcc51d23 100644 (file)
@@ -99,35 +99,54 @@ defmodule Pleroma.Instances.InstanceTest do
     end
   end
 
-  test "Scrapes favicon URLs" do
-    Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} ->
-      %Tesla.Env{
-        status: 200,
-        body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>]
-      }
-    end)
-
-    assert "https://favicon.example.org/favicon.png" ==
-             Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/"))
-  end
+  describe "get_or_update_favicon/1" do
+    test "Scrapes favicon URLs" do
+      Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} ->
+        %Tesla.Env{
+          status: 200,
+          body: ~s[<html><head><link rel="icon" href="/favicon.png"></head></html>]
+        }
+      end)
+
+      assert "https://favicon.example.org/favicon.png" ==
+               Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/"))
+    end
 
-  test "Returns nil on too long favicon URLs" do
-    long_favicon_url =
-      "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png"
-
-    Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} ->
-      %Tesla.Env{
-        status: 200,
-        body: ~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>]
-      }
-    end)
-
-    assert capture_log(fn ->
-             assert nil ==
-                      Instance.get_or_update_favicon(
-                        URI.parse("https://long-favicon.example.org/")
-                      )
-           end) =~
-             "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{"
+    test "Returns nil on too long favicon URLs" do
+      long_favicon_url =
+        "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png"
+
+      Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} ->
+        %Tesla.Env{
+          status: 200,
+          body:
+            ~s[<html><head><link rel="icon" href="] <> long_favicon_url <> ~s["></head></html>]
+        }
+      end)
+
+      assert capture_log(fn ->
+               assert nil ==
+                        Instance.get_or_update_favicon(
+                          URI.parse("https://long-favicon.example.org/")
+                        )
+             end) =~
+               "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{"
+    end
+
+    test "Handles not getting a favicon URL properly" do
+      Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} ->
+        %Tesla.Env{
+          status: 200,
+          body: ~s[<html><head><h1>I wil look down and whisper "GNO.."</h1></head></html>]
+        }
+      end)
+
+      refute capture_log(fn ->
+               assert nil ==
+                        Instance.get_or_update_favicon(
+                          URI.parse("https://no-favicon.example.org/")
+                        )
+             end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: "
+    end
   end
 end
index 17a1e7d661f9234102c28807e54ee0fa7ea8953a..f7f1369e48e35d70a19d925a2a3c2d2f66a05b00 100644 (file)
@@ -1442,7 +1442,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do
   describe "verify_credentials" do
     test "verify_credentials" do
       %{user: user, conn: conn} = oauth_access(["read:accounts"])
-      [notification | _] = insert_list(7, :notification, user: user)
+
+      [notification | _] =
+        insert_list(7, :notification, user: user, activity: insert(:note_activity))
+
       Pleroma.Notification.set_read_up_to(user, notification.id)
       conn = get(conn, "/api/v1/accounts/verify_credentials")
 
index 6dd40fb4a9f741f3b4e613ad3cbd6f20ab029b01..9f0481120df495e61ab4f4812b9ac50beb8c9e4f 100644 (file)
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do
     test "gets markers with correct scopes", %{conn: conn} do
       user = insert(:user)
       token = insert(:oauth_token, user: user, scopes: ["read:statuses"])
-      insert_list(7, :notification, user: user)
+      insert_list(7, :notification, user: user, activity: insert(:note_activity))
 
       {:ok, %{"notifications" => marker}} =
         Pleroma.Marker.upsert(
index 9f22f9dcf99e7a2a2445c8739583098cd595d4d1..c5f491d6bd69da77ab2a7e5584e3bacf14739351 100644 (file)
@@ -448,7 +448,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do
 
     test "shows unread_count only to the account owner" do
       user = insert(:user)
-      insert_list(7, :notification, user: user)
+      insert_list(7, :notification, user: user, activity: insert(:note_activity))
       other_user = insert(:user)
 
       user = User.get_cached_by_ap_id(user.ap_id)
index 21ae35f8b4f99d2015846b66a1bad73533907c57..6d00c2af56ebf181aeb0c8f4ff13e980033babe5 100644 (file)
@@ -56,6 +56,27 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
 
       %{method: :get, url: "http://example.com/error"} ->
         {:error, :overload}
+
+      %{
+        method: :head,
+        url: "http://example.com/huge-page"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
+        }
+
+      %{
+        method: :head,
+        url: "http://example.com/pdf-file"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
+        }
+
+      %{method: :head} ->
+        %Tesla.Env{status: 404, body: "", headers: []}
     end)
 
     :ok
@@ -144,4 +165,12 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
   test "returns error if getting page was not successful" do
     assert {:error, :overload} = Parser.parse("http://example.com/error")
   end
+
+  test "does a HEAD request to check if the body is too large" do
+    assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page")
+  end
+
+  test "does a HEAD request to check if the body is html" do
+    assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file")
+  end
 end