defmodule Mix.Tasks.Pleroma.Database do
alias Pleroma.Conversation
+ alias Pleroma.Hashtag
alias Pleroma.Maintenance
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
+
require Logger
require Pleroma.Constants
+
import Ecto.Query
import Mix.Pleroma
+
use Mix.Task
@shortdoc "A collection of database related tasks"
|> Stream.run()
end
+ def run(["transfer_hashtags"]) do
+ import Ecto.Query
+
+ start_pleroma()
+
+ from(
+ object in Object,
+ left_join: hashtag in assoc(object, :hashtags),
+ where: is_nil(hashtag.id),
+ where: fragment("(?)->>'tag' != '[]'", object.data),
+ select: %{
+ id: object.id,
+ inserted_at: object.inserted_at,
+ tag: fragment("(?)->>'tag'", object.data)
+ },
+ order_by: [desc: object.id]
+ )
+ |> Pleroma.Repo.chunk_stream(100, :batches)
+ |> Stream.each(fn objects ->
+ chunk_start = List.first(objects)
+ chunk_end = List.last(objects)
+
+ Logger.info(
+ "transfer_hashtags: " <>
+ "#{chunk_start.id} (#{chunk_start.inserted_at}) -- " <>
+ "#{chunk_end.id} (#{chunk_end.inserted_at})"
+ )
+
+ Enum.map(
+ objects,
+ fn object ->
+ hashtags =
+ object.tag
+ |> Jason.decode!()
+ |> Enum.filter(&is_bitstring(&1))
+
+ with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
+ Repo.transaction(fn ->
+ for hashtag_record <- hashtag_records do
+ with {:error, _} <-
+ Ecto.Adapters.SQL.query(
+ Repo,
+ "insert into hashtags_objects(hashtag_id, object_id) values " <>
+ "(#{hashtag_record.id}, #{object.id});"
+ ) do
+ Logger.warn(
+ "ERROR: could not link object #{object.id} and hashtag #{hashtag_record.id}"
+ )
+ end
+ end
+ end)
+ else
+ e -> Logger.warn("ERROR: could not process object #{object.id}: #{inspect(e)}")
+ end
+ end
+ )
+ end)
+ |> Stream.run()
+ end
+
def run(["vacuum", args]) do
start_pleroma()
tags
end
- defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
- tags
- |> Enum.filter(&is_bitstring(&1))
- |> Enum.map(fn tag -> "hashtag:" <> tag end)
+ defp hashtags_to_topics(object) do
+ object
+ |> Object.hashtags()
+ |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
end
- defp hashtags_to_topics(_), do: []
-
defp remote_topics(%{local: true}), do: []
defp remote_topics(%{actor: actor}) when is_binary(actor),
--- /dev/null
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Hashtag do
+ use Ecto.Schema
+
+ import Ecto.Changeset
+
+ alias Pleroma.Hashtag
+ alias Pleroma.Repo
+
+ @derive {Jason.Encoder, only: [:data]}
+
+ schema "hashtags" do
+ field(:name, :string)
+ field(:data, :map, default: %{})
+
+ many_to_many(:objects, Pleroma.Object, join_through: "hashtags_objects", on_replace: :delete)
+
+ timestamps()
+ end
+
+ def get_by_name(name) do
+ Repo.get_by(Hashtag, name: name)
+ end
+
+ def get_or_create_by_name(name) when is_bitstring(name) do
+ with %Hashtag{} = hashtag <- get_by_name(name) do
+ {:ok, hashtag}
+ else
+ _ ->
+ %Hashtag{}
+ |> changeset(%{name: name})
+ |> Repo.insert()
+ end
+ end
+
+ def get_or_create_by_names(names) when is_list(names) do
+ Enum.reduce_while(names, {:ok, []}, fn name, {:ok, list} ->
+ case get_or_create_by_name(name) do
+ {:ok, %Hashtag{} = hashtag} ->
+ {:cont, {:ok, list ++ [hashtag]}}
+
+ error ->
+ {:halt, error}
+ end
+ end)
+ end
+
+ def changeset(%Hashtag{} = struct, params) do
+ struct
+ |> cast(params, [:name, :data])
+ |> update_change(:name, &String.downcase/1)
+ |> validate_required([:name])
+ |> unique_constraint(:name)
+ end
+end
alias Pleroma.Activity
alias Pleroma.Config
+ alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.ObjectTombstone
schema "objects" do
field(:data, :map)
+ many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete)
+
timestamps()
end
end
def create(data) do
- Object.change(%Object{}, %{data: data})
+ %Object{}
+ |> Object.change(%{data: data})
|> Repo.insert()
end
|> cast(params, [:data])
|> validate_required([:data])
|> unique_constraint(:ap_id, name: :objects_unique_apid_index)
+ |> maybe_handle_hashtags_change(struct)
+ end
+
+ defp maybe_handle_hashtags_change(changeset, struct) do
+ with data_hashtags_change = get_change(changeset, :data),
+ true <- hashtags_changed?(struct, data_hashtags_change),
+ {:ok, hashtag_records} <-
+ data_hashtags_change
+ |> object_data_hashtags()
+ |> Hashtag.get_or_create_by_names() do
+ put_assoc(changeset, :hashtags, hashtag_records)
+ else
+ false ->
+ changeset
+
+ {:error, hashtag_changeset} ->
+ failed_hashtag = get_field(hashtag_changeset, :name)
+
+ validate_change(changeset, :data, fn _, _ ->
+ [data: "error referencing hashtag: #{failed_hashtag}"]
+ end)
+ end
+ end
+
+ defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do
+ Enum.sort(embedded_hashtags(struct)) !=
+ Enum.sort(object_data_hashtags(data))
end
+ defp hashtags_changed?(_, _), do: false
+
def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id)
def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true))
+
+ def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
+
+ def tags(_), do: []
+
+ def hashtags(object), do: embedded_hashtags(object)
+
+ defp embedded_hashtags(%Object{data: data}) do
+ object_data_hashtags(data)
+ end
+
+ defp embedded_hashtags(_), do: []
+
+ defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
+ # Note: AS2 map-type elements are ignored
+ Enum.filter(tags, &is_bitstring(&1))
+ end
+
+ defp object_data_hashtags(_), do: []
end
defp restrict_since(query, _), do: query
defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ raise_on_missing_preload()
end
- defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
+ defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do
from(
[_activity, object] in query,
where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
)
end
+ defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+ restrict_tag_reject(query, %{tag_reject: [tag_reject]})
+ end
+
defp restrict_tag_reject(query, _), do: query
defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ raise_on_missing_preload()
end
- defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
+ defp restrict_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do
from(
[_activity, object] in query,
where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
)
end
+ defp restrict_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
+ restrict_tag(query, %{tag: tag})
+ end
+
defp restrict_tag_all(query, _), do: query
defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ raise_on_missing_preload()
end
defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
end
defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
+ restrict_tag(query, %{tag: [tag]})
+ end
+
+ defp restrict_tag(query, _), do: query
+
+ defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do
+ if has_named_binding?(query, :thread_mute) do
+ from(
+ [activity, object, thread_mute] in query,
+ group_by: [activity.id, object.id, thread_mute.id]
+ )
+ else
+ from(
+ [activity, object] in query,
+ group_by: [activity.id, object.id]
+ )
+ end
+ |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag)
+ |> having(
+ [hashtag: hashtag],
+ fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject)
+ )
+ end
+
+ defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+ restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
+ end
+
+ defp restrict_hashtag_reject_any(query, _), do: query
+
+ defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do
+ Enum.reduce(
+ tags,
+ query,
+ fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end
+ )
+ end
+
+ defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
+ restrict_hashtag_any(query, %{tag: tag})
+ end
+
+ defp restrict_hashtag_all(query, _), do: query
+
+ defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do
from(
[_activity, object] in query,
- where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
+ join: hashtag in assoc(object, :hashtags),
+ where: hashtag.name in ^tags
)
end
- defp restrict_tag(query, _), do: query
+ defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
+ restrict_hashtag_any(query, %{tag: [tag]})
+ end
+
+ defp restrict_hashtag_any(query, _), do: query
+
+ defp raise_on_missing_preload do
+ raise "Can't use the child object without preloading!"
+ end
defp restrict_recipients(query, [], _user), do: query
skip_thread_containment: Config.get([:instance, :skip_thread_containment])
}
- Activity
- |> maybe_preload_objects(opts)
- |> maybe_preload_bookmarks(opts)
- |> maybe_preload_report_notes(opts)
- |> maybe_set_thread_muted_field(opts)
- |> maybe_order(opts)
- |> restrict_recipients(recipients, opts[:user])
- |> restrict_replies(opts)
- |> restrict_tag(opts)
- |> restrict_tag_reject(opts)
- |> restrict_tag_all(opts)
- |> restrict_since(opts)
- |> restrict_local(opts)
- |> restrict_actor(opts)
- |> restrict_type(opts)
- |> restrict_state(opts)
- |> restrict_favorited_by(opts)
- |> restrict_blocked(restrict_blocked_opts)
- |> restrict_muted(restrict_muted_opts)
- |> restrict_filtered(opts)
- |> restrict_media(opts)
- |> restrict_visibility(opts)
- |> restrict_thread_visibility(opts, config)
- |> restrict_reblogs(opts)
- |> restrict_pinned(opts)
- |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
- |> restrict_instance(opts)
- |> restrict_announce_object_actor(opts)
- |> restrict_filtered(opts)
- |> Activity.restrict_deactivated_users()
- |> exclude_poll_votes(opts)
- |> exclude_chat_messages(opts)
- |> exclude_invisible_actors(opts)
- |> exclude_visibility(opts)
+ query =
+ Activity
+ |> distinct([a], true)
+ |> maybe_preload_objects(opts)
+ |> maybe_preload_bookmarks(opts)
+ |> maybe_preload_report_notes(opts)
+ |> maybe_set_thread_muted_field(opts)
+ |> maybe_order(opts)
+ |> restrict_recipients(recipients, opts[:user])
+ |> restrict_replies(opts)
+ |> restrict_since(opts)
+ |> restrict_local(opts)
+ |> restrict_actor(opts)
+ |> restrict_type(opts)
+ |> restrict_state(opts)
+ |> restrict_favorited_by(opts)
+ |> restrict_blocked(restrict_blocked_opts)
+ |> restrict_muted(restrict_muted_opts)
+ |> restrict_filtered(opts)
+ |> restrict_media(opts)
+ |> restrict_visibility(opts)
+ |> restrict_thread_visibility(opts, config)
+ |> restrict_reblogs(opts)
+ |> restrict_pinned(opts)
+ |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
+ |> restrict_instance(opts)
+ |> restrict_announce_object_actor(opts)
+ |> restrict_filtered(opts)
+ |> Activity.restrict_deactivated_users()
+ |> exclude_poll_votes(opts)
+ |> exclude_chat_messages(opts)
+ |> exclude_invisible_actors(opts)
+ |> exclude_visibility(opts)
+
+ if Config.get([:instance, :improved_hashtag_timeline]) do
+ query
+ |> restrict_hashtag_any(opts)
+ |> restrict_hashtag_all(opts)
+ |> restrict_hashtag_reject_any(opts)
+ else
+ query
+ |> restrict_tag(opts)
+ |> restrict_tag_reject(opts)
+ |> restrict_tag_all(opts)
+ end
end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
object =
if MRF.subdomain_match?(media_nsfw, actor_host) do
- tags = (child_object["tag"] || []) ++ ["nsfw"]
- child_object = Map.put(child_object, "tag", tags)
- child_object = Map.put(child_object, "sensitive", true)
+ child_object =
+ child_object
+ |> Map.put("tag", (child_object["tag"] || []) ++ ["nsfw"])
+ |> Map.put("sensitive", true)
+
Map.put(object, "object", child_object)
else
object
"""
def fix_object(object, options \\ []) do
object
- |> strip_internal_fields
- |> fix_actor
- |> fix_url
- |> fix_attachments
- |> fix_context
+ |> strip_internal_fields()
+ |> fix_actor()
+ |> fix_url()
+ |> fix_attachments()
+ |> fix_context()
|> fix_in_reply_to(options)
- |> fix_emoji
- |> fix_tag
- |> set_sensitive
- |> fix_content_map
- |> fix_addressing
- |> fix_summary
+ |> fix_emoji()
+ |> fix_tag()
+ |> set_sensitive()
+ |> fix_content_map()
+ |> fix_addressing()
+ |> fix_summary()
|> fix_type(options)
end
tags =
tag
|> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
- |> Enum.map(fn %{"name" => name} ->
- name
- |> String.slice(1..-1)
- |> String.downcase()
+ |> Enum.map(fn
+ %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+ %{"name" => hashtag} -> String.downcase(hashtag)
end)
Map.put(object, "tag", tag ++ tags)
%{
activity: activity,
+ object: object,
data: Map.get(object, :data),
actor: actor
}
like_count = object.data["like_count"] || 0
announcement_count = object.data["announcement_count"] || 0
- tags = object.data["tag"] || []
- sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
+ hashtags = Object.hashtags(object)
+ sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw")
+
+ tags = Object.tags(object)
tag_mentions =
tags
<link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
<link><%= @data["external_url"] %></link>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
<% end %>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
--- /dev/null
+defmodule Pleroma.Repo.Migrations.CreateHashtags do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:hashtags) do
+ add(:name, :citext, null: false)
+ add(:data, :map, default: %{})
+
+ timestamps()
+ end
+
+ create_if_not_exists(unique_index(:hashtags, [:name]))
+ end
+end
--- /dev/null
+defmodule Pleroma.Repo.Migrations.CreateHashtagsObjects do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:hashtags_objects) do
+ add(:hashtag_id, references(:hashtags), null: false)
+ add(:object_id, references(:objects), null: false)
+ end
+
+ create_if_not_exists(unique_index(:hashtags_objects, [:hashtag_id, :object_id]))
+ create_if_not_exists(index(:hashtags_objects, [:object_id]))
+ end
+end
defmodule Pleroma.ObjectTest do
use Pleroma.DataCase
use Oban.Testing, repo: Pleroma.Repo
+
import ExUnit.CaptureLog
import Pleroma.Factory
import Tesla.Mock
+
alias Pleroma.Activity
+ alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.Tests.ObanHelpers
assert updated_object.data["like_count"] == 1
end
end
+
+ describe ":hashtags association" do
+ test "Hashtag records are created with Object record and updated on its change" do
+ user = insert(:user)
+
+ {:ok, %{object: object}} =
+ CommonAPI.post(user, %{status: "some text #hashtag1 #hashtag2 ..."})
+
+ assert [%Hashtag{name: "hashtag1"}, %Hashtag{name: "hashtag2"}] =
+ Enum.sort_by(object.hashtags, & &1.name)
+
+ {:ok, object} = Object.update_data(object, %{"tag" => []})
+
+ assert [] = object.hashtags
+
+ object = Object.get_by_id(object.id) |> Repo.preload(:hashtags)
+ assert [] = object.hashtags
+
+ {:ok, object} = Object.update_data(object, %{"tag" => ["abc", "def"]})
+
+ assert [%Hashtag{name: "abc"}, %Hashtag{name: "def"}] =
+ Enum.sort_by(object.hashtags, & &1.name)
+ end
+ end
end
{:ok, status_two} = CommonAPI.post(user, %{status: ". #essais"})
{:ok, status_three} = CommonAPI.post(user, %{status: ". #test #reject"})
- fetch_one = ActivityPub.fetch_activities([], %{type: "Create", tag: "test"})
+ for new_timeline_enabled <- [true, false] do
+ clear_config([:instance, :improved_hashtag_timeline], new_timeline_enabled)
- fetch_two = ActivityPub.fetch_activities([], %{type: "Create", tag: ["test", "essais"]})
+ fetch_one = ActivityPub.fetch_activities([], %{type: "Create", tag: "test"})
- fetch_three =
- ActivityPub.fetch_activities([], %{
- type: "Create",
- tag: ["test", "essais"],
- tag_reject: ["reject"]
- })
+ fetch_two = ActivityPub.fetch_activities([], %{type: "Create", tag: ["test", "essais"]})
- fetch_four =
- ActivityPub.fetch_activities([], %{
- type: "Create",
- tag: ["test"],
- tag_all: ["test", "reject"]
- })
+ fetch_three =
+ ActivityPub.fetch_activities([], %{
+ type: "Create",
+ tag: ["test", "essais"],
+ tag_reject: ["reject"]
+ })
- assert fetch_one == [status_one, status_three]
- assert fetch_two == [status_one, status_two, status_three]
- assert fetch_three == [status_one, status_two]
- assert fetch_four == [status_three]
+ fetch_four =
+ ActivityPub.fetch_activities([], %{
+ type: "Create",
+ tag: ["test"],
+ tag_all: ["test", "reject"]
+ })
+
+ [fetch_one, fetch_two, fetch_three, fetch_four] =
+ Enum.map([fetch_one, fetch_two, fetch_three, fetch_four], fn statuses ->
+ Enum.map(statuses, fn s -> Repo.preload(s, object: :hashtags) end)
+ end)
+
+ assert fetch_one == [status_one, status_three]
+ assert fetch_two == [status_one, status_two, status_three]
+ assert fetch_three == [status_one, status_two]
+ assert fetch_four == [status_three]
+ end
end
describe "insertion" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
- assert "test" in object.data["tag"]
+ assert "test" in Object.tags(object)
+ assert Object.hashtags(object) == ["test"]
end
test "it cleans up incoming notices which are not really DMs" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
- assert Enum.at(object.data["tag"], 2) == "moo"
+ assert Enum.at(Object.tags(object), 2) == "moo"
+ assert Object.hashtags(object) == ["moo"]
end
test "it works for incoming notices with contentMap" do
object = Object.normalize(activity)
- assert object.data["tag"] == ["2hu"]
+ assert Object.tags(object) == ["2hu"]
end
test "it adds emoji in the object" do
mentions: [],
tags: [
%{
- name: "#{object_data["tag"]}",
- url: "/tag/#{object_data["tag"]}"
+ name: "#{hd(object_data["tag"])}",
+ url: "/tag/#{hd(object_data["tag"])}"
}
],
application: %{
projects / akkoma / commitdiff