def run(["import", "users" | _rest]) do
start_pleroma()
- from(u in User, where: not ilike(u.ap_id, "%/relay"))
+ from(u in User, where: u.nickname not in ["internal.fetch", "relay"])
|> get_all(:users)
end
timestamp: user.inserted_at,
instance: URI.parse(user.ap_id).host,
nickname: user.nickname,
- bio: user.bio
+ bio: user.bio,
+ display_name: user.name
}
end
end
defmodule Pleroma.Elasticsearch do
alias Pleroma.Activity
+ alias Pleroma.User
alias Pleroma.Elasticsearch.DocumentMappings
alias Pleroma.Config
+ require Logger
defp url do
Config.get([:elasticsearch, :url])
end
- def put_by_id(id) do
+ defp enabled? do
+ Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch
+ end
+
+ def put_by_id(:activity, id) do
id
|> Activity.get_by_id_with_object()
|> maybe_put_into_elasticsearch()
end
- def maybe_put_into_elasticsearch({:ok, activity}) do
- maybe_put_into_elasticsearch(activity)
+ def maybe_put_into_elasticsearch({:ok, item}) do
+ maybe_put_into_elasticsearch(item)
end
- def maybe_put_into_elasticsearch(%{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity) do
- if Config.get([:search, :provider]) == Pleroma.Search.Elasticsearch do
+ def maybe_put_into_elasticsearch(
+ %{data: %{"type" => "Create"}, object: %{data: %{"type" => "Note"}}} = activity
+ ) do
+ if enabled?() do
actor = Pleroma.Activity.user_actor(activity)
activity
end
end
+ def maybe_put_into_elasticsearch(%User{} = user) do
+ if enabled?() do
+ put(user)
+ end
+ end
+
def maybe_put_into_elasticsearch(_) do
{:ok, :skipped}
end
def put(%Activity{} = activity) do
- Elastix.Document.index(
+ {:ok, _} = Elastix.Document.index(
url(),
"activities",
"activity",
DocumentMappings.Activity.id(activity),
DocumentMappings.Activity.encode(activity)
)
+ {:ok, _} = bulk_post(
+ activity.object.hashtags, :hashtags
+ )
+ end
+
+ def put(%User{} = user) do
+ {:ok, _ } = Elastix.Document.index(
+ url(),
+ "users",
+ "user",
+ DocumentMappings.User.id(user),
+ DocumentMappings.User.encode(user)
+ )
end
def bulk_post(data, :activities) do
d =
data
|> Enum.filter(fn x ->
- t = x.object
- |> Map.get(:data, %{})
- |> Map.get("type", "")
+ t =
+ x.object
+ |> Map.get(:data, %{})
+ |> Map.get("type", "")
+
t == "Note"
end)
|> Enum.map(fn d ->
end)
|> List.flatten()
- Elastix.Bulk.post(
+ {:ok, %{body: %{"errors" => false}}} = Elastix.Bulk.post(
url(),
d,
index: "activities",
)
end
- def search_activities(q) do
- Elastix.Search.search(
- url(),
- "activities",
- ["activity"],
- q
- )
+ def search(:raw, index, type, q) do
+ with {:ok, raw_results} <- Elastix.Search.search(url(), index, [type], q) do
+ results =
+ raw_results
+ |> Map.get(:body, %{})
+ |> Map.get("hits", %{})
+ |> Map.get("hits", [])
+
+ {:ok, results}
+ else
+ {:error, e} ->
+ Logger.error(e)
+ {:error, e}
+ end
+ end
+
+ def search(:activities, q) do
+ with {:ok, results} <- search(:raw, "activities", "activity", q) do
+ results
+ |> Enum.map(fn result -> result["_id"] end)
+ |> Pleroma.Activity.all_by_ids_with_object()
+ else
+ e ->
+ Logger.error(e)
+ []
+ end
+ end
+
+ def search(:users, q) do
+ with {:ok, results} <- search(:raw, "users", "user", q) do
+ results
+ |> Enum.map(fn result -> result["_id"] end)
+ |> Pleroma.User.get_all_by_ids()
+ else
+ e ->
+ Logger.error(e)
+ []
+ end
+ end
+
+ def search(:hashtags, q) do
+ with {:ok, results} <- search(:raw, "hashtags", "hashtag", q) do
+ results
+ |> Enum.map(fn result -> result["_source"]["hashtag"] end)
+ else
+ e ->
+ Logger.error(e)
+ []
+ end
end
end
@behaviour Pleroma.Search
alias Pleroma.Web.MastodonAPI.StatusView
+ alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.ActivityPub.Visibility
+ alias Pleroma.Search.Elasticsearch.Parsers
+ alias Pleroma.Web.Endpoint
- defp to_es(term) when is_binary(term) do
+ defp es_query(:activity, query) do
%{
- match: %{
- content: %{
- query: term,
- operator: "AND"
+ query: %{
+ bool: %{
+ must: Parsers.Activity.parse(query)
}
}
}
end
- defp to_es({:quoted, term}), do: to_es(term)
-
- defp to_es({:filter, ["hashtag", query]}) do
+ defp es_query(:user, query) do
%{
- term: %{
- hashtags: %{
- value: query
+ query: %{
+ bool: %{
+ must: Parsers.User.parse(query)
}
}
}
end
- defp to_es({:filter, [field, query]}) do
+ defp es_query(:hashtag, query) do
%{
- term: %{
- field => %{
- value: query
+ query: %{
+ bool: %{
+ must: Parsers.Hashtag.parse(query)
}
}
}
end
- defp parse(query) do
- query
- |> SearchParser.parse!()
- |> Enum.map(&to_es/1)
- end
-
@impl Pleroma.Search
def search(%{assigns: %{user: user}} = _conn, %{q: query} = _params, _options) do
- q = %{
- query: %{
- bool: %{
- must: parse(String.trim(query))
- }
- }
- }
+ parsed_query =
+ query
+ |> String.trim()
+ |> SearchParser.parse!()
- out = Pleroma.Elasticsearch.search_activities(q)
+ activity_task =
+ Task.async(fn ->
+ q = es_query(:activity, parsed_query)
- with {:ok, raw_results} <- out do
- results =
- raw_results
- |> Map.get(:body, %{})
- |> Map.get("hits", %{})
- |> Map.get("hits", [])
- |> Enum.map(fn result -> result["_id"] end)
- |> Pleroma.Activity.all_by_ids_with_object()
- |> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end)
+ Pleroma.Elasticsearch.search(:activities, q)
+ |> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end)
+ end)
- %{
- "accounts" => [],
- "hashtags" => [],
- "statuses" =>
- StatusView.render("index.json",
- activities: results,
- for: user,
- as: :activity
- )
- }
- end
+ user_task =
+ Task.async(fn ->
+ q = es_query(:user, parsed_query)
+
+ Pleroma.Elasticsearch.search(:users, q)
+ |> Enum.filter(fn x -> Pleroma.User.visible_for(x, user) == :visible end)
+ end)
+
+ hashtag_task =
+ Task.async(fn ->
+ q = es_query(:hashtag, parsed_query)
+
+ Pleroma.Elasticsearch.search(:hashtags, q)
+ end)
+
+ activity_results = Task.await(activity_task)
+ user_results = Task.await(user_task)
+ hashtag_results = Task.await(hashtag_task)
+
+ %{
+ "accounts" =>
+ AccountView.render("index.json",
+ users: user_results,
+ for: user
+ ),
+ "hashtags" =>
+ Enum.map(hashtag_results, fn x ->
+ %{
+ url: Endpoint.url() <> "/tag/" <> x,
+ name: x
+ }
+ end),
+ "statuses" =>
+ StatusView.render("index.json",
+ activities: activity_results,
+ for: user,
+ as: :activity
+ )
+ }
end
end
--- /dev/null
+defmodule Pleroma.Search.Elasticsearch.Parsers.Activity do
+ defp to_es(term) when is_binary(term) do
+ %{
+ match: %{
+ content: %{
+ query: term,
+ operator: "AND"
+ }
+ }
+ }
+ end
+
+ defp to_es({:quoted, term}), do: to_es(term)
+
+ defp to_es({:filter, ["hashtag", query]}) do
+ %{
+ term: %{
+ hashtags: %{
+ value: query
+ }
+ }
+ }
+ end
+
+ defp to_es({:filter, [field, query]}) do
+ %{
+ term: %{
+ field => %{
+ value: query
+ }
+ }
+ }
+ end
+
+ def parse(q) do
+ Enum.map(q, &to_es/1)
+ end
+end
--- /dev/null
+defmodule Pleroma.Search.Elasticsearch.Parsers.Hashtag do
+ defp to_es(term) when is_binary(term) do
+ %{
+ term: %{
+ hashtag: %{
+ value: String.downcase(term),
+ }
+ }
+ }
+ end
+
+ defp to_es({:quoted, term}), do: to_es(term)
+
+ defp to_es({:filter, ["hashtag", query]}) do
+ %{
+ term: %{
+ hashtag: %{
+ value: String.downcase(query)
+ }
+ }
+ }
+ end
+
+ defp to_es({:filter, _}), do: nil
+
+ def parse(q) do
+ Enum.map(q, &to_es/1)
+ |> Enum.filter(fn x -> x != nil end)
+ end
+end
--- /dev/null
+defmodule Pleroma.Search.Elasticsearch.Parsers.User do
+ defp to_es(term) when is_binary(term) do
+ %{
+ bool: %{
+ minimum_should_match: 1,
+ should: [
+ %{
+ match: %{
+ bio: %{
+ query: term,
+ operator: "AND"
+ }
+ }
+ },
+ %{
+ term: %{
+ nickname: %{
+ value: term
+ }
+ }
+ },
+ %{
+ match: %{
+ display_name: %{
+ query: term,
+ operator: "AND"
+ }
+ }
+ }
+ ]
+ }
+ }
+ end
+
+ defp to_es({:quoted, term}), do: to_es(term)
+
+ defp to_es({:filter, ["user", query]}) do
+ %{
+ term: %{
+ nickname: %{
+ value: query
+ }
+ }
+ }
+ end
+
+ defp to_es({:filter, _}), do: nil
+
+ def parse(q) do
+ Enum.map(q, &to_es/1)
+ |> Enum.filter(fn x -> x != nil end)
+ end
+end
def update_and_set_cache(changeset) do
with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do
+ Pleroma.Elasticsearch.maybe_put_into_elasticsearch(user)
set_cache(user)
end
end
@impl true
def handle_after_transaction(%Pleroma.Activity{data: %{"type" => "Create"}} = activity) do
- Pleroma.Elasticsearch.put_by_id(activity.id)
+ Pleroma.Elasticsearch.put_by_id(:activity, activity.id)
end
def handle_after_transaction(%Pleroma.Activity{}) do
projects / akkoma / commitdiff