X-Git-Url: http://git.squeep.com/?a=blobdiff_plain;ds=inline;f=lib%2Fpleroma%2Frepo.ex;h=4524bd5e2c766636382ba2c0c34d409d10c289b4;hb=0d149502fe8397df6f374b44f1b0087d05cadecb;hp=f57e088bc288f321c11c62868ea5b363d77a702b;hpb=1557b99beb3b406572ef2d3baaabed1c9baeca1c;p=akkoma
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index f57e088bc..4524bd5e2 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -1,5 +1,5 @@
# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors
+# Copyright © 2017-2020 Pleroma Authors
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Repo do
@@ -8,9 +8,10 @@ defmodule Pleroma.Repo do
adapter: Ecto.Adapters.Postgres,
migration_timestamps: [type: :naive_datetime_usec]
- defmodule Instrumenter do
- use Prometheus.EctoInstrumenter
- end
+ import Ecto.Query
+ require Logger
+
+ defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
@doc """
Dynamically loads the repository url from the
@@ -47,4 +48,50 @@ defmodule Pleroma.Repo do
_ -> {:error, :not_found}
end
end
+
+ @doc """
+ Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+ `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+ ## Examples
+
+ # fetch records one-by-one
+ iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+ # fetch records in bulk
+ iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+ """
+ @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+ def chunk_stream(query, chunk_size, returns_as \\ :one) do
+ # We don't actually need start and end funcitons of resource streaming,
+ # but it seems to be the only way to not fetch records one-by-one and
+ # have individual records be the elements of the stream, instead of
+ # lists of records
+ Stream.resource(
+ fn -> 0 end,
+ fn
+ last_id ->
+ query
+ |> order_by(asc: :id)
+ |> where([r], r.id > ^last_id)
+ |> limit(^chunk_size)
+ |> all()
+ |> case do
+ [] ->
+ {:halt, last_id}
+
+ records ->
+ last_id = List.last(records).id
+
+ if returns_as == :one do
+ {records, last_id}
+ else
+ {[records], last_id}
+ end
+ end
+ end,
+ fn _ -> :ok end
+ )
+ end
end