Merge remote-tracking branch 'upstream/develop' into block-behavior

[akkoma] / lib / pleroma / repo.ex

diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index f57e088bc288f321c11c62868ea5b363d77a702b..4524bd5e2c766636382ba2c0c34d409d10c289b4 100644 (file)
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Repo do
@@ -8,9 +8,10 @@ defmodule Pleroma.Repo do
     adapter: Ecto.Adapters.Postgres,
     migration_timestamps: [type: :naive_datetime_usec]
 
-  defmodule Instrumenter do
-    use Prometheus.EctoInstrumenter
-  end
+  import Ecto.Query
+  require Logger
+
+  defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
 
   @doc """
   Dynamically loads the repository url from the
@@ -47,4 +48,50 @@ defmodule Pleroma.Repo do
       _ -> {:error, :not_found}
     end
   end
+
+  @doc """
+  Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+  `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+  ## Examples
+
+  # fetch records one-by-one
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+  # fetch records in bulk
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+  """
+  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+  def chunk_stream(query, chunk_size, returns_as \\ :one) do
+    # We don't actually need start and end funcitons of resource streaming,
+    # but it seems to be the only way to not fetch records one-by-one and
+    # have individual records be the elements of the stream, instead of
+    # lists of records
+    Stream.resource(
+      fn -> 0 end,
+      fn
+        last_id ->
+          query
+          |> order_by(asc: :id)
+          |> where([r], r.id > ^last_id)
+          |> limit(^chunk_size)
+          |> all()
+          |> case do
+            [] ->
+              {:halt, last_id}
+
+            records ->
+              last_id = List.last(records).id
+
+              if returns_as == :one do
+                {records, last_id}
+              else
+                {[records], last_id}
+              end
+          end
+      end,
+      fn _ -> :ok end
+    )
+  end
 end