From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 26 Dec 2020 19:23:05 +0000 (+0300)
Subject: Merge remote-tracking branch 'remotes/origin/develop' into feature/object-hashtags... 
X-Git-Url: http://git.squeep.com/?a=commitdiff_plain;h=4134abef63e1165f5701741c1012e64cb908654c;hp=e4f1d8f48c85b8a388d6c3945db157de5ce588c5;p=akkoma

Merge remote-tracking branch 'remotes/origin/develop' into feature/object-hashtags-rework
---

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 22151ce08..093c7dd30 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -4,14 +4,18 @@
 
 defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.Conversation
+  alias Pleroma.Hashtag
   alias Pleroma.Maintenance
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+
   require Logger
   require Pleroma.Constants
+
   import Ecto.Query
   import Mix.Pleroma
+
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
@@ -128,6 +132,66 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> Stream.run()
   end
 
+  def run(["transfer_hashtags"]) do
+    import Ecto.Query
+
+    start_pleroma()
+
+    from(
+      object in Object,
+      left_join: hashtag in assoc(object, :hashtags),
+      where: is_nil(hashtag.id),
+      where: fragment("(?)->>'tag' != '[]'", object.data),
+      select: %{
+        id: object.id,
+        inserted_at: object.inserted_at,
+        tag: fragment("(?)->>'tag'", object.data)
+      },
+      order_by: [desc: object.id]
+    )
+    |> Pleroma.Repo.chunk_stream(100, :batches)
+    |> Stream.each(fn objects ->
+      chunk_start = List.first(objects)
+      chunk_end = List.last(objects)
+
+      Logger.info(
+        "transfer_hashtags: " <>
+          "#{chunk_start.id} (#{chunk_start.inserted_at}) -- " <>
+          "#{chunk_end.id} (#{chunk_end.inserted_at})"
+      )
+
+      Enum.map(
+        objects,
+        fn object ->
+          hashtags =
+            object.tag
+            |> Jason.decode!()
+            |> Enum.filter(&is_bitstring(&1))
+
+          with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
+            Repo.transaction(fn ->
+              for hashtag_record <- hashtag_records do
+                with {:error, _} <-
+                       Ecto.Adapters.SQL.query(
+                         Repo,
+                         "insert into hashtags_objects(hashtag_id, object_id) values " <>
+                           "(#{hashtag_record.id}, #{object.id});"
+                       ) do
+                  Logger.warn(
+                    "ERROR: could not link object #{object.id} and hashtag #{hashtag_record.id}"
+                  )
+                end
+              end
+            end)
+          else
+            e -> Logger.warn("ERROR: could not process object #{object.id}: #{inspect(e)}")
+          end
+        end
+      )
+    end)
+    |> Stream.run()
+  end
+
   def run(["vacuum", args]) do
     start_pleroma()
 
diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
index fe2e8cb5c..2c74ac2bf 100644
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do
     tags
   end
 
-  defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
-    tags
-    |> Enum.filter(&is_bitstring(&1))
-    |> Enum.map(fn tag -> "hashtag:" <> tag end)
+  defp hashtags_to_topics(object) do
+    object
+    |> Object.hashtags()
+    |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
   end
 
-  defp hashtags_to_topics(_), do: []
-
   defp remote_topics(%{local: true}), do: []
 
   defp remote_topics(%{actor: actor}) when is_binary(actor),
diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex
new file mode 100644
index 000000000..b05927563
--- /dev/null
+++ b/lib/pleroma/hashtag.ex
@@ -0,0 +1,58 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Hashtag do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  alias Pleroma.Hashtag
+  alias Pleroma.Repo
+
+  @derive {Jason.Encoder, only: [:data]}
+
+  schema "hashtags" do
+    field(:name, :string)
+    field(:data, :map, default: %{})
+
+    many_to_many(:objects, Pleroma.Object, join_through: "hashtags_objects", on_replace: :delete)
+
+    timestamps()
+  end
+
+  def get_by_name(name) do
+    Repo.get_by(Hashtag, name: name)
+  end
+
+  def get_or_create_by_name(name) when is_bitstring(name) do
+    with %Hashtag{} = hashtag <- get_by_name(name) do
+      {:ok, hashtag}
+    else
+      _ ->
+        %Hashtag{}
+        |> changeset(%{name: name})
+        |> Repo.insert()
+    end
+  end
+
+  def get_or_create_by_names(names) when is_list(names) do
+    Enum.reduce_while(names, {:ok, []}, fn name, {:ok, list} ->
+      case get_or_create_by_name(name) do
+        {:ok, %Hashtag{} = hashtag} ->
+          {:cont, {:ok, list ++ [hashtag]}}
+
+        error ->
+          {:halt, error}
+      end
+    end)
+  end
+
+  def changeset(%Hashtag{} = struct, params) do
+    struct
+    |> cast(params, [:name, :data])
+    |> update_change(:name, &String.downcase/1)
+    |> validate_required([:name])
+    |> unique_constraint(:name)
+  end
+end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index b4a994da9..1d756bcd1 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Object do
 
   alias Pleroma.Activity
   alias Pleroma.Config
+  alias Pleroma.Hashtag
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
   alias Pleroma.ObjectTombstone
@@ -28,6 +29,8 @@ defmodule Pleroma.Object do
   schema "objects" do
     field(:data, :map)
 
+    many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete)
+
     timestamps()
   end
 
@@ -49,7 +52,8 @@ defmodule Pleroma.Object do
   end
 
   def create(data) do
-    Object.change(%Object{}, %{data: data})
+    %Object{}
+    |> Object.change(%{data: data})
     |> Repo.insert()
   end
 
@@ -58,8 +62,37 @@ defmodule Pleroma.Object do
     |> cast(params, [:data])
     |> validate_required([:data])
     |> unique_constraint(:ap_id, name: :objects_unique_apid_index)
+    |> maybe_handle_hashtags_change(struct)
+  end
+
+  defp maybe_handle_hashtags_change(changeset, struct) do
+    with data_hashtags_change = get_change(changeset, :data),
+         true <- hashtags_changed?(struct, data_hashtags_change),
+         {:ok, hashtag_records} <-
+           data_hashtags_change
+           |> object_data_hashtags()
+           |> Hashtag.get_or_create_by_names() do
+      put_assoc(changeset, :hashtags, hashtag_records)
+    else
+      false ->
+        changeset
+
+      {:error, hashtag_changeset} ->
+        failed_hashtag = get_field(hashtag_changeset, :name)
+
+        validate_change(changeset, :data, fn _, _ ->
+          [data: "error referencing hashtag: #{failed_hashtag}"]
+        end)
+    end
+  end
+
+  defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do
+    Enum.sort(embedded_hashtags(struct)) !=
+      Enum.sort(object_data_hashtags(data))
   end
 
+  defp hashtags_changed?(_, _), do: false
+
   def get_by_id(nil), do: nil
   def get_by_id(id), do: Repo.get(Object, id)
 
@@ -346,4 +379,23 @@ defmodule Pleroma.Object do
 
   def self_replies(object, opts \\ []),
     do: replies(object, Keyword.put(opts, :self_only, true))
+
+  def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
+
+  def tags(_), do: []
+
+  def hashtags(object), do: embedded_hashtags(object)
+
+  defp embedded_hashtags(%Object{data: data}) do
+    object_data_hashtags(data)
+  end
+
+  defp embedded_hashtags(_), do: []
+
+  defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
+    # Note: AS2 map-type elements are ignored
+    Enum.filter(tags, &is_bitstring(&1))
+  end
+
+  defp object_data_hashtags(_), do: []
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5059bff03..54d1a2350 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -663,33 +663,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_since(query, _), do: query
 
   defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
-    raise "Can't use the child object without preloading!"
+    raise_on_missing_preload()
   end
 
-  defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
+  defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do
     from(
       [_activity, object] in query,
       where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
     )
   end
 
+  defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+    restrict_tag_reject(query, %{tag_reject: [tag_reject]})
+  end
+
   defp restrict_tag_reject(query, _), do: query
 
   defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
-    raise "Can't use the child object without preloading!"
+    raise_on_missing_preload()
   end
 
-  defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
+  defp restrict_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do
     from(
       [_activity, object] in query,
       where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
     )
   end
 
+  defp restrict_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
+    restrict_tag(query, %{tag: tag})
+  end
+
   defp restrict_tag_all(query, _), do: query
 
   defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
-    raise "Can't use the child object without preloading!"
+    raise_on_missing_preload()
   end
 
   defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
@@ -700,13 +708,79 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
+    restrict_tag(query, %{tag: [tag]})
+  end
+
+  defp restrict_tag(query, _), do: query
+
+  defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do
+    if has_named_binding?(query, :thread_mute) do
+      from(
+        [activity, object, thread_mute] in query,
+        group_by: [activity.id, object.id, thread_mute.id]
+      )
+    else
+      from(
+        [activity, object] in query,
+        group_by: [activity.id, object.id]
+      )
+    end
+    |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag)
+    |> having(
+      [hashtag: hashtag],
+      fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject)
+    )
+  end
+
+  defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+    restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
+  end
+
+  defp restrict_hashtag_reject_any(query, _), do: query
+
+  defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do
+    Enum.reduce(
+      tags,
+      query,
+      fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end
+    )
+  end
+
+  defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
+    restrict_hashtag_any(query, %{tag: tag})
+  end
+
+  defp restrict_hashtag_all(query, _), do: query
+
+  defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
+    raise_on_missing_preload()
+  end
+
+  defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do
     from(
       [_activity, object] in query,
-      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
+      join: hashtag in assoc(object, :hashtags),
+      where: hashtag.name in ^tags
     )
   end
 
-  defp restrict_tag(query, _), do: query
+  defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
+    restrict_hashtag_any(query, %{tag: [tag]})
+  end
+
+  defp restrict_hashtag_any(query, _), do: query
+
+  defp raise_on_missing_preload do
+    raise "Can't use the child object without preloading!"
+  end
 
   defp restrict_recipients(query, [], _user), do: query
 
@@ -1091,40 +1165,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       skip_thread_containment: Config.get([:instance, :skip_thread_containment])
     }
 
-    Activity
-    |> maybe_preload_objects(opts)
-    |> maybe_preload_bookmarks(opts)
-    |> maybe_preload_report_notes(opts)
-    |> maybe_set_thread_muted_field(opts)
-    |> maybe_order(opts)
-    |> restrict_recipients(recipients, opts[:user])
-    |> restrict_replies(opts)
-    |> restrict_tag(opts)
-    |> restrict_tag_reject(opts)
-    |> restrict_tag_all(opts)
-    |> restrict_since(opts)
-    |> restrict_local(opts)
-    |> restrict_actor(opts)
-    |> restrict_type(opts)
-    |> restrict_state(opts)
-    |> restrict_favorited_by(opts)
-    |> restrict_blocked(restrict_blocked_opts)
-    |> restrict_muted(restrict_muted_opts)
-    |> restrict_filtered(opts)
-    |> restrict_media(opts)
-    |> restrict_visibility(opts)
-    |> restrict_thread_visibility(opts, config)
-    |> restrict_reblogs(opts)
-    |> restrict_pinned(opts)
-    |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
-    |> restrict_instance(opts)
-    |> restrict_announce_object_actor(opts)
-    |> restrict_filtered(opts)
-    |> Activity.restrict_deactivated_users()
-    |> exclude_poll_votes(opts)
-    |> exclude_chat_messages(opts)
-    |> exclude_invisible_actors(opts)
-    |> exclude_visibility(opts)
+    query =
+      Activity
+      |> distinct([a], true)
+      |> maybe_preload_objects(opts)
+      |> maybe_preload_bookmarks(opts)
+      |> maybe_preload_report_notes(opts)
+      |> maybe_set_thread_muted_field(opts)
+      |> maybe_order(opts)
+      |> restrict_recipients(recipients, opts[:user])
+      |> restrict_replies(opts)
+      |> restrict_since(opts)
+      |> restrict_local(opts)
+      |> restrict_actor(opts)
+      |> restrict_type(opts)
+      |> restrict_state(opts)
+      |> restrict_favorited_by(opts)
+      |> restrict_blocked(restrict_blocked_opts)
+      |> restrict_muted(restrict_muted_opts)
+      |> restrict_filtered(opts)
+      |> restrict_media(opts)
+      |> restrict_visibility(opts)
+      |> restrict_thread_visibility(opts, config)
+      |> restrict_reblogs(opts)
+      |> restrict_pinned(opts)
+      |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
+      |> restrict_instance(opts)
+      |> restrict_announce_object_actor(opts)
+      |> restrict_filtered(opts)
+      |> Activity.restrict_deactivated_users()
+      |> exclude_poll_votes(opts)
+      |> exclude_chat_messages(opts)
+      |> exclude_invisible_actors(opts)
+      |> exclude_visibility(opts)
+
+    if Config.get([:instance, :improved_hashtag_timeline]) do
+      query
+      |> restrict_hashtag_any(opts)
+      |> restrict_hashtag_all(opts)
+      |> restrict_hashtag_reject_any(opts)
+    else
+      query
+      |> restrict_tag(opts)
+      |> restrict_tag_reject(opts)
+      |> restrict_tag_all(opts)
+    end
   end
 
   def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 6cd91826d..e92091d66 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -74,9 +74,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
 
     object =
       if MRF.subdomain_match?(media_nsfw, actor_host) do
-        tags = (child_object["tag"] || []) ++ ["nsfw"]
-        child_object = Map.put(child_object, "tag", tags)
-        child_object = Map.put(child_object, "sensitive", true)
+        child_object =
+          child_object
+          |> Map.put("tag", (child_object["tag"] || []) ++ ["nsfw"])
+          |> Map.put("sensitive", true)
+
         Map.put(object, "object", child_object)
       else
         object
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 565d32433..fd17793d0 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -32,18 +32,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   def fix_object(object, options \\ []) do
     object
-    |> strip_internal_fields
-    |> fix_actor
-    |> fix_url
-    |> fix_attachments
-    |> fix_context
+    |> strip_internal_fields()
+    |> fix_actor()
+    |> fix_url()
+    |> fix_attachments()
+    |> fix_context()
     |> fix_in_reply_to(options)
-    |> fix_emoji
-    |> fix_tag
-    |> set_sensitive
-    |> fix_content_map
-    |> fix_addressing
-    |> fix_summary
+    |> fix_emoji()
+    |> fix_tag()
+    |> set_sensitive()
+    |> fix_content_map()
+    |> fix_addressing()
+    |> fix_summary()
     |> fix_type(options)
   end
 
@@ -315,10 +315,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     tags =
       tag
       |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
-      |> Enum.map(fn %{"name" => name} ->
-        name
-        |> String.slice(1..-1)
-        |> String.downcase()
+      |> Enum.map(fn
+        %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+        %{"name" => hashtag} -> String.downcase(hashtag)
       end)
 
     Map.put(object, "tag", tag ++ tags)
diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex
index 30e0a2a55..1155c6a39 100644
--- a/lib/pleroma/web/feed/feed_view.ex
+++ b/lib/pleroma/web/feed/feed_view.ex
@@ -32,6 +32,7 @@ defmodule Pleroma.Web.Feed.FeedView do
 
     %{
       activity: activity,
+      object: object,
       data: Map.get(object, :data),
       actor: actor
     }
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 2301e21cf..bd08aa203 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -201,8 +201,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     like_count = object.data["like_count"] || 0
     announcement_count = object.data["announcement_count"] || 0
 
-    tags = object.data["tag"] || []
-    sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
+    hashtags = Object.hashtags(object)
+    sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw")
+
+    tags = Object.tags(object)
 
     tag_mentions =
       tags
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
index 3fd150c4e..6688830ba 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
@@ -22,7 +22,7 @@
     <link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
   <% end %>
 
-  <%= for tag <- @data["tag"] || [] do %>
+  <%= for tag <- Pleroma.Object.hashtags(@object) do %>
     <category term="<%= tag %>"></category>
   <% end %>
 
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
index 42960de7d..fc6d74b42 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
@@ -21,7 +21,7 @@
     <link><%= @data["external_url"] %></link>
   <% end %>
 
-  <%= for tag <- @data["tag"] || [] do %>
+  <%= for tag <- Pleroma.Object.hashtags(@object) do %>
     <category term="<%= tag %>"></category>
   <% end %>
 
diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
index cf5874a91..c2de28fe4 100644
--- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
@@ -41,7 +41,7 @@
       <% end %>
     <% end %>
 
-    <%= for tag <- @data["tag"] || [] do %>
+    <%= for tag <- Pleroma.Object.hashtags(@object) do %>
       <category term="<%= tag %>"></category>
     <% end %>
 
diff --git a/priv/repo/migrations/20201221202251_create_hashtags.exs b/priv/repo/migrations/20201221202251_create_hashtags.exs
new file mode 100644
index 000000000..afc522002
--- /dev/null
+++ b/priv/repo/migrations/20201221202251_create_hashtags.exs
@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.CreateHashtags do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:hashtags) do
+      add(:name, :citext, null: false)
+      add(:data, :map, default: %{})
+
+      timestamps()
+    end
+
+    create_if_not_exists(unique_index(:hashtags, [:name]))
+  end
+end
diff --git a/priv/repo/migrations/20201221203824_create_hashtags_objects.exs b/priv/repo/migrations/20201221203824_create_hashtags_objects.exs
new file mode 100644
index 000000000..b2649b4fb
--- /dev/null
+++ b/priv/repo/migrations/20201221203824_create_hashtags_objects.exs
@@ -0,0 +1,13 @@
+defmodule Pleroma.Repo.Migrations.CreateHashtagsObjects do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:hashtags_objects) do
+      add(:hashtag_id, references(:hashtags), null: false)
+      add(:object_id, references(:objects), null: false)
+    end
+
+    create_if_not_exists(unique_index(:hashtags_objects, [:hashtag_id, :object_id]))
+    create_if_not_exists(index(:hashtags_objects, [:object_id]))
+  end
+end
diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs
index 5d4e6fb84..819ecd210 100644
--- a/test/pleroma/object_test.exs
+++ b/test/pleroma/object_test.exs
@@ -5,10 +5,13 @@
 defmodule Pleroma.ObjectTest do
   use Pleroma.DataCase
   use Oban.Testing, repo: Pleroma.Repo
+
   import ExUnit.CaptureLog
   import Pleroma.Factory
   import Tesla.Mock
+
   alias Pleroma.Activity
+  alias Pleroma.Hashtag
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.Tests.ObanHelpers
@@ -406,4 +409,28 @@ defmodule Pleroma.ObjectTest do
       assert updated_object.data["like_count"] == 1
     end
   end
+
+  describe ":hashtags association" do
+    test "Hashtag records are created with Object record and updated on its change" do
+      user = insert(:user)
+
+      {:ok, %{object: object}} =
+        CommonAPI.post(user, %{status: "some text #hashtag1 #hashtag2 ..."})
+
+      assert [%Hashtag{name: "hashtag1"}, %Hashtag{name: "hashtag2"}] =
+               Enum.sort_by(object.hashtags, & &1.name)
+
+      {:ok, object} = Object.update_data(object, %{"tag" => []})
+
+      assert [] = object.hashtags
+
+      object = Object.get_by_id(object.id) |> Repo.preload(:hashtags)
+      assert [] = object.hashtags
+
+      {:ok, object} = Object.update_data(object, %{"tag" => ["abc", "def"]})
+
+      assert [%Hashtag{name: "abc"}, %Hashtag{name: "def"}] =
+               Enum.sort_by(object.hashtags, & &1.name)
+    end
+  end
 end
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 9eb7ae86b..573b26d66 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -199,28 +199,37 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
     {:ok, status_two} = CommonAPI.post(user, %{status: ". #essais"})
     {:ok, status_three} = CommonAPI.post(user, %{status: ". #test #reject"})
 
-    fetch_one = ActivityPub.fetch_activities([], %{type: "Create", tag: "test"})
+    for new_timeline_enabled <- [true, false] do
+      clear_config([:instance, :improved_hashtag_timeline], new_timeline_enabled)
 
-    fetch_two = ActivityPub.fetch_activities([], %{type: "Create", tag: ["test", "essais"]})
+      fetch_one = ActivityPub.fetch_activities([], %{type: "Create", tag: "test"})
 
-    fetch_three =
-      ActivityPub.fetch_activities([], %{
-        type: "Create",
-        tag: ["test", "essais"],
-        tag_reject: ["reject"]
-      })
+      fetch_two = ActivityPub.fetch_activities([], %{type: "Create", tag: ["test", "essais"]})
 
-    fetch_four =
-      ActivityPub.fetch_activities([], %{
-        type: "Create",
-        tag: ["test"],
-        tag_all: ["test", "reject"]
-      })
+      fetch_three =
+        ActivityPub.fetch_activities([], %{
+          type: "Create",
+          tag: ["test", "essais"],
+          tag_reject: ["reject"]
+        })
 
-    assert fetch_one == [status_one, status_three]
-    assert fetch_two == [status_one, status_two, status_three]
-    assert fetch_three == [status_one, status_two]
-    assert fetch_four == [status_three]
+      fetch_four =
+        ActivityPub.fetch_activities([], %{
+          type: "Create",
+          tag: ["test"],
+          tag_all: ["test", "reject"]
+        })
+
+      [fetch_one, fetch_two, fetch_three, fetch_four] =
+        Enum.map([fetch_one, fetch_two, fetch_three, fetch_four], fn statuses ->
+          Enum.map(statuses, fn s -> Repo.preload(s, object: :hashtags) end)
+        end)
+
+      assert fetch_one == [status_one, status_three]
+      assert fetch_two == [status_one, status_two, status_three]
+      assert fetch_three == [status_one, status_two]
+      assert fetch_four == [status_three]
+    end
   end
 
   describe "insertion" do
diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
index b4a006aec..a33959d9f 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
@@ -39,7 +39,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
       object = Object.normalize(data["object"])
 
-      assert "test" in object.data["tag"]
+      assert "test" in Object.tags(object)
+      assert Object.hashtags(object) == ["test"]
     end
 
     test "it cleans up incoming notices which are not really DMs" do
@@ -220,7 +221,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
       object = Object.normalize(data["object"])
 
-      assert Enum.at(object.data["tag"], 2) == "moo"
+      assert Enum.at(Object.tags(object), 2) == "moo"
+      assert Object.hashtags(object) == ["moo"]
     end
 
     test "it works for incoming notices with contentMap" do
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
index 585b2c174..1e98208fb 100644
--- a/test/pleroma/web/common_api_test.exs
+++ b/test/pleroma/web/common_api_test.exs
@@ -493,7 +493,7 @@ defmodule Pleroma.Web.CommonAPITest do
 
     object = Object.normalize(activity)
 
-    assert object.data["tag"] == ["2hu"]
+    assert Object.tags(object) == ["2hu"]
   end
 
   test "it adds emoji in the object" do
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
index fa9066716..22717ce90 100644
--- a/test/pleroma/web/mastodon_api/views/status_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -262,8 +262,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
       mentions: [],
       tags: [
         %{
-          name: "#{object_data["tag"]}",
-          url: "/tag/#{object_data["tag"]}"
+          name: "#{hd(object_data["tag"])}",
+          url: "/tag/#{hd(object_data["tag"])}"
         }
       ],
       application: %{