- Pleroma.Repo.chunk_stream(
- from(Pleroma.Object,
- # Only index public posts which are notes and have some text
- where:
- fragment("data->>'type' = 'Note'") and
- fragment("LENGTH(data->>'source') > 0") and
- fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public())
- ),
- 200,
- :batches
- )
- |> Stream.map(fn objects ->
- Enum.map(objects, fn object ->
- data = object.data
- %{id: object.id, source: data["source"], ap: data["id"]}
- end)
- end)
- |> Stream.each(fn objects ->
- {:ok, _} =
- Pleroma.HTTP.post(
- "#{endpoint}/indexes/objects/documents",
- Jason.encode!(objects)
+ {:ok, _} =
+ meili_put(
+ "/indexes/objects/settings/searchable-attributes",
+ [
+ "content"
+ ]
+ )
+
+ IO.puts("Created indices. Starting to insert posts.")
+
+ chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size])
+
+ Pleroma.Repo.transaction(
+ fn ->
+ query =
+ from(Pleroma.Object,
+ # Only index public and unlisted posts which are notes and have some text
+ where:
+ fragment("data->>'type' = 'Note'") and
+ (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or
+ fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())),
+ order_by: [desc: fragment("data->'published'")]
+ )
+
+ count = query |> Pleroma.Repo.aggregate(:count, :data)
+ IO.puts("Entries to index: #{count}")
+
+ Pleroma.Repo.stream(
+ query,
+ timeout: :infinity