From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 09:22:32 +0000 (+0300)
Subject: Merge branch 'develop' into feature/database-compaction
X-Git-Url: http://git.squeep.com/?a=commitdiff_plain;h=627e5a0a4992cc19fc65a7e93a09c470c8e2bf33;p=akkoma

Merge branch 'develop' into feature/database-compaction
---

627e5a0a4992cc19fc65a7e93a09c470c8e2bf33
diff --cc lib/mix/tasks/compact_database.ex
index 7de50812a,000000000..17b9721f7
mode 100644,000000..100644
--- a/lib/mix/tasks/compact_database.ex
+++ b/lib/mix/tasks/compact_database.ex
@@@ -1,57 -1,0 +1,57 @@@
 +defmodule Mix.Tasks.CompactDatabase do
 +  @moduledoc """
 +  Compact the database by flattening the object graph.
 +  """
 +
 +  require Logger
 +
 +  use Mix.Task
-   import Mix.Ecto
 +  import Ecto.Query
-   alias Pleroma.{Repo, Object, Activity}
++  alias Pleroma.Activity
++  alias Pleroma.Repo
 +
 +  defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do
 +    data =
 +      activity.data
 +      |> Map.put("object", object_id)
 +
 +    {:ok, activity} =
 +      Activity.change(activity, %{data: data})
 +      |> Repo.update()
 +
 +    {:ok, activity}
 +  end
 +
 +  defp maybe_compact(%Activity{} = activity), do: {:ok, activity}
 +
 +  defp activity_query(min_id, max_id) do
 +    from(
 +      a in Activity,
 +      where: fragment("?->>'type' = 'Create'", a.data),
 +      where: a.id >= ^min_id,
 +      where: a.id < ^max_id
 +    )
 +  end
 +
-   def run(args) do
++  def run(_args) do
 +    Application.ensure_all_started(:pleroma)
 +
 +    max = Repo.aggregate(Activity, :max, :id)
 +    Logger.info("Considering #{max} activities")
 +
 +    chunks = 0..round(max / 100)
 +
 +    Enum.each(chunks, fn i ->
 +      min = i * 100
 +      max = min + 100
 +
 +      activity_query(min, max)
 +      |> Repo.all()
 +      |> Enum.each(&maybe_compact/1)
 +
 +      IO.write(".")
 +    end)
 +
 +    Logger.info("Finished.")
 +  end
 +end
diff --cc lib/pleroma/activity.ex
index e3aa4eb97,e6507e5ca..99cc9c077
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@@ -1,7 -1,31 +1,32 @@@
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
  defmodule Pleroma.Activity do
    use Ecto.Schema
-   alias Pleroma.{Repo, Activity, Notification, Object}
-   import Ecto.{Query, Changeset}
+ 
+   alias Pleroma.Activity
+   alias Pleroma.Notification
+   alias Pleroma.Object
+   alias Pleroma.Repo
+ 
++  import Ecto.Changeset
+   import Ecto.Query
+ 
+   @type t :: %__MODULE__{}
+   @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+ 
+   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
+   @mastodon_notification_types %{
+     "Create" => "mention",
+     "Follow" => "follow",
+     "Announce" => "reblog",
+     "Like" => "favourite"
+   }
+ 
+   @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
+                                          into: %{},
+                                          do: {v, k}
  
    schema "activities" do
      field(:data, :map)
@@@ -22,17 -79,45 +80,52 @@@
      )
    end
  
 +  def change(struct, params \\ %{}) do
 +    struct
 +    |> cast(params, [:data])
 +    |> validate_required([:data])
 +    |> unique_constraint(:ap_id, name: :activities_unique_apid_index)
 +  end
 +
-   # TODO:
-   # Go through these and fix them everywhere.
-   # Wrong name, only returns create activities
-   def all_by_object_ap_id_q(ap_id) do
+   def get_by_ap_id_with_object(ap_id) do
+     Repo.one(
+       from(
+         activity in Activity,
+         where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)),
+         left_join: o in Object,
+         on:
+           fragment(
+             "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+             o.data,
+             activity.data,
+             activity.data
+           ),
+         preload: [object: o]
+       )
+     )
+   end
+ 
+   def get_by_id(id) do
+     Repo.get(Activity, id)
+   end
+ 
+   def get_by_id_with_object(id) do
+     from(activity in Activity,
+       where: activity.id == ^id,
+       inner_join: o in Object,
+       on:
+         fragment(
+           "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
+           o.data,
+           activity.data,
+           activity.data
+         ),
+       preload: [object: o]
+     )
+     |> Repo.one()
+   end
+ 
+   def by_object_ap_id(ap_id) do
      from(
        activity in Activity,
        where:
@@@ -84,19 -202,97 +210,101 @@@
      |> Repo.one()
    end
  
-   def get_create_activity_by_object_ap_id(_), do: nil
- 
-   def normalize(obj) when is_map(obj), do: normalize(obj["id"])
-   def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id(ap_id)
 -  def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
 -  def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
--  def normalize(_), do: nil
- 
 +  defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
-     get_create_activity_by_object_ap_id(ap_id)
++    get_create_by_object_ap_id_with_object(ap_id)
 +  end
 +
 +  defp get_in_reply_to_activity_from_object(_), do: nil
  
 -  def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
 -    get_create_by_object_ap_id(ap_id)
 +  def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do
 +    get_in_reply_to_activity_from_object(Object.normalize(object))
    end
+ 
 -  def get_in_reply_to_activity(_), do: nil
++  def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
++  def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
++  def normalize(_), do: nil
+ 
+   def delete_by_ap_id(id) when is_binary(id) do
+     by_object_ap_id(id)
+     |> select([u], u)
+     |> Repo.delete_all()
+     |> elem(1)
+     |> Enum.find(fn
+       %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
+       _ -> nil
+     end)
+   end
+ 
+   def delete_by_ap_id(_), do: nil
+ 
+   for {ap_type, type} <- @mastodon_notification_types do
+     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
+       do: unquote(type)
+   end
+ 
+   def mastodon_notification_type(%Activity{}), do: nil
+ 
+   def from_mastodon_notification_type(type) do
+     Map.get(@mastodon_to_ap_notification_types, type)
+   end
+ 
+   def all_by_actor_and_id(actor, status_ids \\ [])
+   def all_by_actor_and_id(_actor, []), do: []
+ 
+   def all_by_actor_and_id(actor, status_ids) do
+     Activity
+     |> where([s], s.id in ^status_ids)
+     |> where([s], s.actor == ^actor)
+     |> Repo.all()
+   end
+ 
+   def increase_replies_count(nil), do: nil
+ 
+   def increase_replies_count(object_ap_id) do
+     from(a in create_by_object_ap_id(object_ap_id),
+       update: [
+         set: [
+           data:
+             fragment(
+               """
+               jsonb_set(?, '{object, repliesCount}',
+                 (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
+               """,
+               a.data,
+               a.data
+             )
+         ]
+       ]
+     )
+     |> Repo.update_all([])
+     |> case do
+       {1, [activity]} -> activity
+       _ -> {:error, "Not found"}
+     end
+   end
+ 
+   def decrease_replies_count(nil), do: nil
+ 
+   def decrease_replies_count(object_ap_id) do
+     from(a in create_by_object_ap_id(object_ap_id),
+       update: [
+         set: [
+           data:
+             fragment(
+               """
+               jsonb_set(?, '{object, repliesCount}',
+                 (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true)
+               """,
+               a.data,
+               a.data
+             )
+         ]
+       ]
+     )
+     |> Repo.update_all([])
+     |> case do
+       {1, [activity]} -> activity
+       _ -> {:error, "Not found"}
+     end
+   end
  end
diff --cc lib/pleroma/gopher/server.ex
index 1ab15611c,6a56a6f67..2ebc5d5f7
--- a/lib/pleroma/gopher/server.ex
+++ b/lib/pleroma/gopher/server.ex
@@@ -32,12 -36,11 +36,12 @@@ defmodule Pleroma.Gopher.Server d
  end
  
  defmodule Pleroma.Gopher.Server.ProtocolHandler do
-   alias Pleroma.Web.ActivityPub.ActivityPub
-   alias Pleroma.User
    alias Pleroma.Activity
-   alias Pleroma.Repo
    alias Pleroma.HTML
 +  alias Pleroma.Object
+   alias Pleroma.User
+   alias Pleroma.Web.ActivityPub.ActivityPub
+   alias Pleroma.Web.ActivityPub.Visibility
  
    def start_link(ref, socket, transport, opts) do
      pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
diff --cc lib/pleroma/html.ex
index 1b920d7fd,7f1dbe28c..4b42d8c9b
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@@ -17,14 -24,67 +24,68 @@@ defmodule Pleroma.HTML d
      end)
    end
  
-   def filter_tags(html, scrubber) do
-     html |> Scrubber.scrub(scrubber)
+   def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
+   def filter_tags(html), do: filter_tags(html, nil)
+   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
+ 
+   def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
+     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
+ 
+     Cachex.fetch!(:scrubber_cache, key, fn _key ->
 -      ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false)
++      object = Pleroma.Object.normalize(activity)
++      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false)
+     end)
+   end
+ 
+   def get_cached_stripped_html_for_activity(content, activity, key) do
+     get_cached_scrubbed_html_for_activity(
+       content,
+       HtmlSanitizeEx.Scrubber.StripTags,
+       activity,
+       key
+     )
    end
  
-   def filter_tags(html), do: filter_tags(html, nil)
+   def ensure_scrubbed_html(
+         content,
+         scrubbers,
+         false = _fake
+       ) do
+     {:commit, filter_tags(content, scrubbers)}
+   end
+ 
+   def ensure_scrubbed_html(
+         content,
+         scrubbers,
+         true = _fake
+       ) do
+     {:ignore, filter_tags(content, scrubbers)}
+   end
+ 
+   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
+     generate_scrubber_signature([scrubber])
+   end
+ 
+   defp generate_scrubber_signature(scrubbers) do
+     Enum.reduce(scrubbers, "", fn scrubber, signature ->
+       "#{signature}#{to_string(scrubber)}"
+     end)
+   end
  
-   def strip_tags(html) do
-     html |> Scrubber.scrub(Scrubber.StripTags)
+   def extract_first_external_url(_, nil), do: {:error, "No content"}
+ 
+   def extract_first_external_url(object, content) do
+     key = "URL|#{object.id}"
+ 
+     Cachex.fetch!(:scrubber_cache, key, fn _key ->
+       result =
+         content
+         |> Floki.filter_out("a.mention")
+         |> Floki.attribute("a", "href")
+         |> Enum.at(0)
+ 
+       {:commit, {:ok, result}}
+     end)
    end
  end
  
diff --cc lib/pleroma/object.ex
index 0e9aefb63,013d62157..3f1d0fea1
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@@ -1,8 -1,20 +1,21 @@@
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
  defmodule Pleroma.Object do
    use Ecto.Schema
-   alias Pleroma.{Repo, Object, Activity}
+ 
+   alias Pleroma.Activity
+   alias Pleroma.Object
 +  alias Pleroma.Object.Fetcher
-   import Ecto.{Query, Changeset}
+   alias Pleroma.ObjectTombstone
+   alias Pleroma.Repo
+   alias Pleroma.User
+ 
+   import Ecto.Query
+   import Ecto.Changeset
+ 
+   require Logger
  
    schema "objects" do
      field(:data, :map)
@@@ -28,31 -40,61 +41,63 @@@
      Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
    end
  
 +  def normalize(_, fetch_remote \\ true)
+   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
+   # Use this whenever possible, especially when walking graphs in an O(N) loop!
 -  def normalize(%Activity{object: %Object{} = object}), do: object
++  def normalize(%Activity{object: %Object{} = object}, _), do: object
  
-   def normalize(obj, fetch_remote) when is_map(obj), do: normalize(obj["id"], fetch_remote)
-   def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id)
+   # A hack for fake activities
 -  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}) do
++  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do
+     %Object{id: "pleroma:fake_object_id", data: data}
+   end
+ 
+   # Catch and log Object.normalize() calls where the Activity's child object is not
+   # preloaded.
 -  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
++  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do
+     Logger.debug(
+       "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+     )
+ 
+     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+ 
 -    normalize(ap_id)
++    normalize(ap_id, fetch_remote)
+   end
+ 
 -  def normalize(%Activity{data: %{"object" => ap_id}}) do
++  def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do
+     Logger.debug(
+       "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
+     )
+ 
+     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
+ 
 -    normalize(ap_id)
++    normalize(ap_id, fetch_remote)
+   end
+ 
+   # Old way, try fetching the object through cache.
 -  def normalize(%{"id" => ap_id}), do: normalize(ap_id)
 -  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
 -  def normalize(_), do: nil
++  def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote)
 +  def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
-   def normalize(obj, _), do: nil
++  def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id)
++  def normalize(_, _), do: nil
  
-   if Mix.env() == :test do
-     def get_cached_by_ap_id(ap_id) do
-       get_by_ap_id(ap_id)
-     end
-   else
-     def get_cached_by_ap_id(ap_id) do
-       key = "object:#{ap_id}"
- 
-       Cachex.fetch!(:object_cache, key, fn _ ->
-         object = get_by_ap_id(ap_id)
- 
-         if object do
-           {:commit, object}
-         else
-           {:ignore, object}
-         end
-       end)
-     end
+   # Owned objects can only be mutated by their owner
+   def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),
+     do: actor == ap_id
+ 
+   # Legacy objects can be mutated by anybody
+   def authorize_mutation(%Object{}, %User{}), do: true
+ 
+   def get_cached_by_ap_id(ap_id) do
+     key = "object:#{ap_id}"
+ 
+     Cachex.fetch!(:object_cache, key, fn _ ->
+       object = get_by_ap_id(ap_id)
+ 
+       if object do
+         {:commit, object}
+       else
+         {:ignore, object}
+       end
+     end)
    end
  
    def context_mapping(context) do
diff --cc lib/pleroma/object/containment.ex
index 010b768bd,000000000..27e89d87f
mode 100644,000000..100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@@ -1,64 -1,0 +1,64 @@@
 +defmodule Pleroma.Object.Containment do
 +  @moduledoc """
 +  # Object Containment
 +
 +  This module contains some useful functions for containing objects to specific
 +  origins and determining those origins.  They previously lived in the
 +  ActivityPub `Transmogrifier` module.
 +
 +  Object containment is an important step in validating remote objects to prevent
 +  spoofing, therefore removal of object containment functions is NOT recommended.
 +  """
 +
 +  require Logger
 +
 +  def get_actor(%{"actor" => actor}) when is_binary(actor) do
 +    actor
 +  end
 +
 +  def get_actor(%{"actor" => actor}) when is_list(actor) do
 +    if is_binary(Enum.at(actor, 0)) do
 +      Enum.at(actor, 0)
 +    else
 +      Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
 +      |> Map.get("id")
 +    end
 +  end
 +
 +  def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do
 +    id
 +  end
 +
 +  def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
 +    get_actor(%{"actor" => actor})
 +  end
 +
 +  @doc """
 +  Checks that an imported AP object's actor matches the domain it came from.
 +  """
-   def contain_origin(id, %{"actor" => nil}), do: :error
++  def contain_origin(_id, %{"actor" => nil}), do: :error
 +
-   def contain_origin(id, %{"actor" => actor} = params) do
++  def contain_origin(id, %{"actor" => _actor} = params) do
 +    id_uri = URI.parse(id)
 +    actor_uri = URI.parse(get_actor(params))
 +
 +    if id_uri.host == actor_uri.host do
 +      :ok
 +    else
 +      :error
 +    end
 +  end
 +
-   def contain_origin_from_id(id, %{"id" => nil}), do: :error
++  def contain_origin_from_id(_id, %{"id" => nil}), do: :error
 +
-   def contain_origin_from_id(id, %{"id" => other_id} = params) do
++  def contain_origin_from_id(id, %{"id" => other_id} = _params) do
 +    id_uri = URI.parse(id)
 +    other_uri = URI.parse(other_id)
 +
 +    if id_uri.host == other_uri.host do
 +      :ok
 +    else
 +      :error
 +    end
 +  end
 +end
diff --cc lib/pleroma/object/fetcher.ex
index c98722f39,000000000..19d9c51af
mode 100644,000000..100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@@ -1,78 -1,0 +1,78 @@@
 +defmodule Pleroma.Object.Fetcher do
-   alias Pleroma.{Object, Repo}
++  alias Pleroma.Object
 +  alias Pleroma.Object.Containment
 +  alias Pleroma.Web.ActivityPub.Transmogrifier
 +  alias Pleroma.Web.OStatus
 +
 +  require Logger
 +
 +  @httpoison Application.get_env(:pleroma, :httpoison)
 +
 +  # TODO:
 +  # This will create a Create activity, which we need internally at the moment.
 +  def fetch_object_from_id(id) do
 +    if object = Object.get_cached_by_ap_id(id) do
 +      {:ok, object}
 +    else
 +      Logger.info("Fetching #{id} via AP")
 +
 +      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
 +           nil <- Object.normalize(data, false),
 +           params <- %{
 +             "type" => "Create",
 +             "to" => data["to"],
 +             "cc" => data["cc"],
 +             "actor" => data["actor"] || data["attributedTo"],
 +             "object" => data
 +           },
 +           :ok <- Containment.contain_origin(id, params),
 +           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
 +        {:ok, Object.normalize(activity.data["object"], false)}
 +      else
 +        {:error, {:reject, nil}} ->
 +          {:reject, nil}
 +
 +        object = %Object{} ->
 +          {:ok, object}
 +
 +        _e ->
 +          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 +
 +          case OStatus.fetch_activity_from_url(id) do
 +            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"], false)}
 +            e -> e
 +          end
 +      end
 +    end
 +  end
 +
 +  def fetch_object_from_id!(id) do
 +    with {:ok, object} <- fetch_object_from_id(id) do
 +      object
 +    else
 +      _e ->
 +        nil
 +    end
 +  end
 +
 +  def fetch_and_contain_remote_object_from_id(id) do
 +    Logger.info("Fetching #{id} via AP")
 +
 +    with true <- String.starts_with?(id, "http"),
 +         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
 +           @httpoison.get(
 +             id,
 +             [Accept: "application/activity+json"],
 +             follow_redirect: true,
 +             timeout: 10000,
 +             recv_timeout: 20000
 +           ),
 +         {:ok, data} <- Jason.decode(body),
 +         :ok <- Containment.contain_origin_from_id(id, data) do
 +      {:ok, data}
 +    else
 +      e ->
 +        {:error, e}
 +    end
 +  end
 +end
diff --cc lib/pleroma/web/activity_pub/activity_pub.ex
index fefefc320,54dd4097c..1a3b47cb3
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@@ -1,12 -1,26 +1,26 @@@
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
  defmodule Pleroma.Web.ActivityPub.ActivityPub do
-   alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+   alias Pleroma.Activity
+   alias Pleroma.Instances
+   alias Pleroma.Notification
+   alias Pleroma.Object
 +  alias Pleroma.Object.Fetcher
-   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
-   alias Pleroma.Web.WebFinger
+   alias Pleroma.Pagination
+   alias Pleroma.Repo
+   alias Pleroma.Upload
+   alias Pleroma.User
+   alias Pleroma.Web.ActivityPub.MRF
+   alias Pleroma.Web.ActivityPub.Transmogrifier
    alias Pleroma.Web.Federator
--  alias Pleroma.Web.OStatus
+   alias Pleroma.Web.WebFinger
+ 
    import Ecto.Query
    import Pleroma.Web.ActivityPub.Utils
+   import Pleroma.Web.ActivityPub.Visibility
+ 
    require Logger
  
    @httpoison Application.get_env(:pleroma, :httpoison)
@@@ -53,14 -75,53 +75,53 @@@
      end
    end
  
-   def insert(map, local \\ true) when is_map(map) do
+   defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do
+     limit = Pleroma.Config.get([:instance, :remote_limit])
+     String.length(content) <= limit
+   end
+ 
+   defp check_remote_limit(_), do: true
+ 
+   def increase_note_count_if_public(actor, object) do
+     if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
+   end
+ 
+   def decrease_note_count_if_public(actor, object) do
+     if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor}
+   end
+ 
+   def increase_replies_count_if_reply(%{
+         "object" => %{"inReplyTo" => reply_ap_id} = object,
+         "type" => "Create"
+       }) do
+     if is_public?(object) do
+       Activity.increase_replies_count(reply_ap_id)
+       Object.increase_replies_count(reply_ap_id)
+     end
+   end
+ 
+   def increase_replies_count_if_reply(_create_data), do: :noop
+ 
+   def decrease_replies_count_if_reply(%Object{
+         data: %{"inReplyTo" => reply_ap_id} = object
+       }) do
+     if is_public?(object) do
+       Activity.decrease_replies_count(reply_ap_id)
+       Object.decrease_replies_count(reply_ap_id)
+     end
+   end
+ 
+   def decrease_replies_count_if_reply(_object), do: :noop
+ 
+   def insert(map, local \\ true, fake \\ false) when is_map(map) do
      with nil <- Activity.normalize(map),
-          map <- lazy_put_activity_defaults(map),
+          map <- lazy_put_activity_defaults(map, fake),
           :ok <- check_actor_is_active(map["actor"]),
+          {_, true} <- {:remote_limit_error, check_remote_limit(map)},
           {:ok, map} <- MRF.filter(map),
-          {:ok, map} <- insert_full_object(map) do
-       {recipients, _, _} = get_recipients(map)
- 
+          {recipients, _, _} = get_recipients(map),
+          {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
 -         {:ok, object} <- insert_full_object(map) do
++         {:ok, map, object} <- insert_full_object(map) do
        {:ok, activity} =
          Repo.insert(%Activity{
            data: map,
@@@ -81,9 -169,7 +169,8 @@@
    def stream_out(activity) do
      public = "https://www.w3.org/ns/activitystreams#Public"
  
 -    if activity.data["type"] in ["Create", "Announce", "Delete"] do
 +    if activity.data["type"] in ["Create", "Announce"] do
 +      object = Object.normalize(activity.data["object"])
- 
        Pleroma.Web.Streamer.stream("user", activity)
        Pleroma.Web.Streamer.stream("list", activity)
  
@@@ -94,16 -180,18 +181,18 @@@
            Pleroma.Web.Streamer.stream("public:local", activity)
          end
  
-         object.data
-         |> Map.get("tag", [])
-         |> Enum.filter(fn tag -> is_bitstring(tag) end)
-         |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
+         if activity.data["type"] in ["Create"] do
 -          activity.data["object"]
++          object.data
+           |> Map.get("tag", [])
+           |> Enum.filter(fn tag -> is_bitstring(tag) end)
+           |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
  
-         if object.data["attachment"] != [] do
-           Pleroma.Web.Streamer.stream("public:media", activity)
 -          if activity.data["object"]["attachment"] != [] do
++          if object.data["attachment"] != [] do
+             Pleroma.Web.Streamer.stream("public:media", activity)
  
-           if activity.local do
-             Pleroma.Web.Streamer.stream("public:local:media", activity)
+             if activity.local do
+               Pleroma.Web.Streamer.stream("public:local:media", activity)
+             end
            end
          end
        else
diff --cc lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 7b7c0e090,3331ebebd..0b80566bf
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@@ -1,11 -1,20 +1,21 @@@
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
  defmodule Pleroma.Web.ActivityPub.ActivityPubController do
    use Pleroma.Web, :controller
-   alias Pleroma.{User, Object}
+ 
+   alias Pleroma.Activity
+   alias Pleroma.Object
 +  alias Pleroma.Object.Fetcher
-   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
+   alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
+   alias Pleroma.Web.ActivityPub.ObjectView
    alias Pleroma.Web.ActivityPub.Relay
+   alias Pleroma.Web.ActivityPub.Transmogrifier
+   alias Pleroma.Web.ActivityPub.UserView
    alias Pleroma.Web.ActivityPub.Utils
+   alias Pleroma.Web.ActivityPub.Visibility
    alias Pleroma.Web.Federator
  
    require Logger
diff --cc lib/pleroma/web/activity_pub/transmogrifier.ex
index c4567193f,39cd31921..0637b18dc
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@@ -2,13 -6,13 +6,16 @@@ defmodule Pleroma.Web.ActivityPub.Trans
    @moduledoc """
    A module to handle coding from internal to wire ActivityPub and back.
    """
 +  alias Pleroma.User
 +  alias Pleroma.Object
-   alias Pleroma.Object.{Containment, Fetcher}
++  alias Pleroma.Object.Containment
    alias Pleroma.Activity
+   alias Pleroma.Object
    alias Pleroma.Repo
+   alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
    alias Pleroma.Web.ActivityPub.Utils
+   alias Pleroma.Web.ActivityPub.Visibility
  
    import Ecto.Query
  
@@@ -50,14 -188,14 +141,14 @@@
  
    def fix_actor(%{"attributedTo" => actor} = object) do
      object
 -    |> Map.put("actor", get_actor(%{"actor" => actor}))
 +    |> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
    end
  
-   def fix_likes(%{"likes" => likes} = object)
-       when is_bitstring(likes) do
-     # Check for standardisation
-     # This is what Peertube does
-     # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+   # Check for standardisation
+   # This is what Peertube does
+   # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+   # Prismo returns only an integer (count) as "likes"
+   def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
      object
      |> Map.put("likes", [])
      |> Map.put("like_count", 0)
@@@ -85,10 -223,10 +176,10 @@@
            ""
        end
  
 -    case fetch_obj_helper(in_reply_to_id) do
 +    case get_obj_helper(in_reply_to_id) do
        {:ok, replied_object} ->
-         with %Activity{} = activity <-
-                Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do
+         with %Activity{} = _activity <-
+                Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
            object
            |> Map.put("inReplyTo", replied_object.data["id"])
            |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
@@@ -282,36 -503,10 +456,10 @@@
      end
    end
  
-   defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do
-     with true <- id =~ "follows",
-          %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id),
-          %Activity{} = activity <- Utils.fetch_latest_follow(follower, followed) do
-       {:ok, activity}
-     else
-       _ -> {:error, nil}
-     end
-   end
- 
-   defp mastodon_follow_hack(_), do: {:error, nil}
- 
-   defp get_follow_activity(follow_object, followed) do
-     with object_id when not is_nil(object_id) <- Utils.get_ap_id(follow_object),
-          {_, %Activity{} = activity} <- {:activity, Activity.get_by_ap_id(object_id)} do
-       {:ok, activity}
-     else
-       # Can't find the activity. This might a Mastodon 2.3 "Accept"
-       {:activity, nil} ->
-         mastodon_follow_hack(follow_object, followed)
- 
-       _ ->
-         {:error, nil}
-     end
-   end
- 
    def handle_incoming(
-         %{"type" => "Accept", "object" => follow_object, "actor" => actor, "id" => id} = data
+         %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
        ) do
 -    with actor <- get_actor(data),
 +    with actor <- Containment.get_actor(data),
           %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
           {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
           {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
@@@ -335,9 -530,9 +483,9 @@@
    end
  
    def handle_incoming(
-         %{"type" => "Reject", "object" => follow_object, "actor" => actor, "id" => id} = data
+         %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
        ) do
 -    with actor <- get_actor(data),
 +    with actor <- Containment.get_actor(data),
           %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
           {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
           {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
@@@ -359,11 -554,11 +507,11 @@@
    end
  
    def handle_incoming(
-         %{"type" => "Like", "object" => object_id, "actor" => actor, "id" => id} = data
+         %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
        ) do
 -    with actor <- get_actor(data),
 +    with actor <- Containment.get_actor(data),
           %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
 -         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
 +         {:ok, object} <- get_obj_helper(object_id),
           {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
        {:ok, activity}
      else
@@@ -372,12 -567,13 +520,13 @@@
    end
  
    def handle_incoming(
-         %{"type" => "Announce", "object" => object_id, "actor" => actor, "id" => id} = data
+         %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
        ) do
 -    with actor <- get_actor(data),
 +    with actor <- Containment.get_actor(data),
           %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
 -         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
 +         {:ok, object} <- get_obj_helper(object_id),
-          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do
+          public <- Visibility.is_public?(data),
+          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
        {:ok, activity}
      else
        _e -> :error
diff --cc lib/pleroma/web/activity_pub/utils.ex
index bc5b98f1a,ccc9da7c6..581b9d1ab
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@@ -180,18 -234,14 +234,18 @@@ defmodule Pleroma.Web.ActivityPub.Util
    @doc """
    Inserts a full object if it is contained in an activity.
    """
 -  def insert_full_object(%{"object" => %{"type" => type} = object_data})
 +  def insert_full_object(%{"object" => %{"type" => type} = object_data} = map)
        when is_map(object_data) and type in @supported_object_types do
      with {:ok, object} <- Object.create(object_data) do
 -      {:ok, object}
 +      map =
 +        map
 +        |> Map.put("object", object.data["id"])
 +
-       {:ok, map}
++      {:ok, map, object}
      end
    end
  
-   def insert_full_object(map), do: {:ok, map}
 -  def insert_full_object(_), do: {:ok, nil}
++  def insert_full_object(map), do: {:ok, map, nil}
  
    def update_object_in_activities(%{data: %{"id" => id}} = object) do
      # TODO
diff --cc lib/pleroma/web/common_api/common_api.ex
index c83f8a6a9,74babdf14..9c3daac2c
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@@ -63,11 -124,8 +124,11 @@@ defmodule Pleroma.Web.CommonAPI d
        nil ->
          "public"
  
-       inReplyTo ->
+       in_reply_to ->
 -        Pleroma.Web.MastodonAPI.StatusView.get_visibility(in_reply_to.data["object"])
 +        # XXX: these heuristics should be moved out of MastodonAPI.
-         with %Object{} = object <- Object.normalize(inReplyTo.data["object"]) do
++        with %Object{} = object <- Object.normalize(in_reply_to) do
 +          Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data)
 +        end
      end
    end
  
diff --cc lib/pleroma/web/common_api/utils.ex
index ec66452c2,185292878..7781f1635
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@@ -177,11 -226,8 +226,10 @@@ defmodule Pleroma.Web.CommonAPI.Utils d
      }
  
      if inReplyTo do
 +      inReplyToObject = Object.normalize(inReplyTo.data["object"])
 +
        object
 -      |> Map.put("inReplyTo", inReplyTo.data["object"]["id"])
 +      |> Map.put("inReplyTo", inReplyToObject.data["id"])
-       |> Map.put("inReplyToStatusId", inReplyTo.id)
      else
        object
      end
diff --cc lib/pleroma/web/federator/federator.ex
index 0644f8d0a,c47328e13..a1f6373a4
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@@ -9,7 -9,13 +9,14 @@@ defmodule Pleroma.Web.Federator d
    alias Pleroma.Web.ActivityPub.Relay
    alias Pleroma.Web.ActivityPub.Transmogrifier
    alias Pleroma.Web.ActivityPub.Utils
+   alias Pleroma.Web.ActivityPub.Visibility
+   alias Pleroma.Web.Federator.RetryQueue
    alias Pleroma.Web.OStatus
++  alias Pleroma.Object.Containment
+   alias Pleroma.Web.Salmon
+   alias Pleroma.Web.WebFinger
+   alias Pleroma.Web.Websub
+ 
    require Logger
  
    @websub Application.get_env(:pleroma, :websub)
diff --cc lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 71390be0d,63fadce38..24a2d4cb9
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@@ -1,16 -1,43 +1,43 @@@
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
  defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
    use Pleroma.Web, :controller
-   alias Pleroma.{Repo, Object, Activity, User, Notification, Stats}
 -
 +  alias Pleroma.Object.Fetcher
+   alias Ecto.Changeset
+   alias Pleroma.Activity
+   alias Pleroma.Config
+   alias Pleroma.Filter
+   alias Pleroma.Notification
+   alias Pleroma.Object
+   alias Pleroma.Pagination
+   alias Pleroma.Repo
+   alias Pleroma.ScheduledActivity
+   alias Pleroma.Stats
+   alias Pleroma.User
    alias Pleroma.Web
-   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView}
    alias Pleroma.Web.ActivityPub.ActivityPub
-   alias Pleroma.Web.ActivityPub.Utils
+   alias Pleroma.Web.ActivityPub.Visibility
    alias Pleroma.Web.CommonAPI
-   alias Pleroma.Web.OAuth.{Authorization, Token, App}
+   alias Pleroma.Web.MastodonAPI.AccountView
+   alias Pleroma.Web.MastodonAPI.AppView
+   alias Pleroma.Web.MastodonAPI.FilterView
+   alias Pleroma.Web.MastodonAPI.ListView
+   alias Pleroma.Web.MastodonAPI.MastodonAPI
+   alias Pleroma.Web.MastodonAPI.MastodonView
+   alias Pleroma.Web.MastodonAPI.NotificationView
+   alias Pleroma.Web.MastodonAPI.ReportView
+   alias Pleroma.Web.MastodonAPI.ScheduledActivityView
+   alias Pleroma.Web.MastodonAPI.StatusView
    alias Pleroma.Web.MediaProxy
-   alias Comeonin.Pbkdf2
+   alias Pleroma.Web.OAuth.App
+   alias Pleroma.Web.OAuth.Authorization
+   alias Pleroma.Web.OAuth.Token
+ 
+   import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
    import Ecto.Query
+ 
    require Logger
  
    @httpoison Application.get_env(:pleroma, :httpoison)
@@@ -465,22 -681,26 +681,28 @@@
    end
  
    def favourited_by(conn, %{"id" => id}) do
 -    with %Activity{data: %{"object" => %{"likes" => likes}}} <- Activity.get_by_id(id) do
 +    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
 +         %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
        q = from(u in User, where: u.ap_id in ^likes)
        users = Repo.all(q)
-       render(conn, AccountView, "accounts.json", %{users: users, as: :user})
+ 
+       conn
+       |> put_view(AccountView)
+       |> render(AccountView, "accounts.json", %{users: users, as: :user})
      else
        _ -> json(conn, [])
      end
    end
  
    def reblogged_by(conn, %{"id" => id}) do
 -    with %Activity{data: %{"object" => %{"announcements" => announces}}} <- Activity.get_by_id(id) do
 +    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
 +         %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
        q = from(u in User, where: u.ap_id in ^announces)
        users = Repo.all(q)
-       render(conn, AccountView, "accounts.json", %{users: users, as: :user})
+ 
+       conn
+       |> put_view(AccountView)
+       |> render("accounts.json", %{users: users, as: :user})
      else
        _ -> json(conn, [])
      end
@@@ -656,11 -966,41 +968,41 @@@
      json(conn, %{})
    end
  
-   def status_search(query) do
+   def subscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+     with %User{} = subscription_target <- User.get_cached_by_id(id),
+          {:ok, subscription_target} = User.subscribe(user, subscription_target) do
+       conn
+       |> put_view(AccountView)
+       |> render("relationship.json", %{user: user, target: subscription_target})
+     else
+       {:error, message} ->
+         conn
+         |> put_resp_content_type("application/json")
+         |> send_resp(403, Jason.encode!(%{"error" => message}))
+     end
+   end
+ 
+   def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+     with %User{} = subscription_target <- User.get_cached_by_id(id),
+          {:ok, subscription_target} = User.unsubscribe(user, subscription_target) do
+       conn
+       |> put_view(AccountView)
+       |> render("relationship.json", %{user: user, target: subscription_target})
+     else
+       {:error, message} ->
+         conn
+         |> put_resp_content_type("application/json")
+         |> send_resp(403, Jason.encode!(%{"error" => message}))
+     end
+   end
+ 
+   def status_search(user, query) do
      fetched =
        if Regex.match?(~r/https?:/, query) do
-         with {:ok, object} <- Fetcher.fetch_object_from_id(query) do
-           [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
 -        with {:ok, object} <- ActivityPub.fetch_object_from_id(query),
++        with {:ok, object} <- Fetcher.fetch_object_from_id(query),
+              %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+              true <- Visibility.visible_for_user?(activity, user) do
+           [activity]
          else
            _e -> []
          end
diff --cc lib/pleroma/web/mastodon_api/views/status_view.ex
index 31f4675c3,a9f607aa5..e4de5ecfb
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@@ -1,11 -1,19 +1,20 @@@
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
  defmodule Pleroma.Web.MastodonAPI.StatusView do
    use Pleroma.Web, :view
-   alias Pleroma.Web.MastodonAPI.{AccountView, StatusView}
-   alias Pleroma.{User, Activity, Object}
+ 
+   alias Pleroma.Activity
+   alias Pleroma.HTML
+   alias Pleroma.Repo
++  alias Pleroma.Object
+   alias Pleroma.User
+   alias Pleroma.Web.CommonAPI
    alias Pleroma.Web.CommonAPI.Utils
+   alias Pleroma.Web.MastodonAPI.AccountView
+   alias Pleroma.Web.MastodonAPI.StatusView
    alias Pleroma.Web.MediaProxy
-   alias Pleroma.Repo
-   alias Pleroma.HTML
  
    # TODO: Add cached version.
    defp get_replied_to_activities(activities) do
@@@ -19,14 -26,39 +28,40 @@@
          nil
      end)
      |> Enum.filter(& &1)
-     |> Activity.create_activity_by_object_id_query()
+     |> Activity.create_by_object_ap_id()
      |> Repo.all()
      |> Enum.reduce(%{}, fn activity, acc ->
 -      Map.put(acc, activity.data["object"]["id"], activity)
 +      object = Object.normalize(activity.data["object"])
 +      Map.put(acc, object.data["id"], activity)
      end)
    end
  
+   defp get_user(ap_id) do
+     cond do
+       user = User.get_cached_by_ap_id(ap_id) ->
+         user
+ 
+       user = User.get_by_guessed_nickname(ap_id) ->
+         user
+ 
+       true ->
+         User.error_user(ap_id)
+     end
+   end
+ 
+   defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
+     do: context_id
+ 
+   defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
+     do: Utils.context_to_conversation_id(context)
+ 
+   defp get_context_id(_), do: nil
+ 
+   defp reblogged?(activity, user) do
 -    object = activity.data["object"] || %{}
 -    present?(user && user.ap_id in (object["announcements"] || []))
++    object = Object.normalize(activity) || %{}
++    present?(user && user.ap_id in (object.data["announcements"] || []))
+   end
+ 
    def render("index.json", opts) do
      replied_to_activities = get_replied_to_activities(opts.activities)
  
@@@ -87,15 -123,13 +126,15 @@@
    end
  
    def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do
 +    object = Object.normalize(object)
 +
-     user = User.get_cached_by_ap_id(activity.data["actor"])
+     user = get_user(activity.data["actor"])
  
 -    like_count = object["like_count"] || 0
 -    announcement_count = object["announcement_count"] || 0
 +    like_count = object.data["like_count"] || 0
 +    announcement_count = object.data["announcement_count"] || 0
  
 -    tags = object["tag"] || []
 -    sensitive = object["sensitive"] || Enum.member?(tags, "nsfw")
 +    tags = object.data["tag"] || []
 +    sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
  
      mentions =
        activity.recipients
@@@ -103,63 -137,99 +142,101 @@@
        |> Enum.filter(& &1)
        |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end)
  
-     repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || [])
 -    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
 -    bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks
 +    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 +
++    bookmarked = opts[:for] && object.data["id"] in opts[:for].bookmarks
+ 
 -    attachment_data = object["attachment"] || []
 +    attachment_data = object.data["attachment"] || []
-     attachment_data = attachment_data ++ if object.data["type"] == "Video", do: [object], else: []
      attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
  
 -    created_at = Utils.to_masto_date(object["published"])
 +    created_at = Utils.to_masto_date(object.data["published"])
  
      reply_to = get_reply_to(activity, opts)
-     reply_to_user = reply_to && User.get_cached_by_ap_id(reply_to.data["actor"])
 +
-     emojis =
-       (object.data["emoji"] || [])
-       |> Enum.map(fn {name, url} ->
-         name = HTML.strip_tags(name)
- 
-         url =
-           HTML.strip_tags(url)
-           |> MediaProxy.url()
- 
-         %{shortcode: name, url: url, static_url: url, visible_in_picker: false}
-       end)
+     reply_to_user = reply_to && get_user(reply_to.data["actor"])
  
      content =
-       render_content(object.data)
-       |> HTML.filter_tags(User.html_filter_policy(opts[:for]))
+       object
+       |> render_content()
+ 
+     content_html =
+       content
+       |> HTML.get_cached_scrubbed_html_for_activity(
+         User.html_filter_policy(opts[:for]),
+         activity,
+         "mastoapi:content"
+       )
+ 
+     content_plaintext =
+       content
+       |> HTML.get_cached_stripped_html_for_activity(
+         activity,
+         "mastoapi:content"
+       )
+ 
 -    summary = object["summary"] || ""
++    summary = object.data["summary"] || ""
+ 
+     summary_html =
+       summary
+       |> HTML.get_cached_scrubbed_html_for_activity(
+         User.html_filter_policy(opts[:for]),
+         activity,
+         "mastoapi:summary"
+       )
+ 
+     summary_plaintext =
+       summary
+       |> HTML.get_cached_stripped_html_for_activity(
+         activity,
+         "mastoapi:summary"
+       )
+ 
+     card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
+ 
+     url =
+       if user.local do
+         Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity)
+       else
 -        object["external_url"] || object["id"]
++        object.data["external_url"] || object.data["id"]
+       end
  
      %{
        id: to_string(activity.id),
 -      uri: object["id"],
 +      uri: object.data["id"],
-       url: object.data["external_url"] || object.data["id"],
+       url: url,
        account: AccountView.render("account.json", %{user: user}),
        in_reply_to_id: reply_to && to_string(reply_to.id),
        in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id),
        reblog: nil,
-       content: content,
+       card: card,
+       content: content_html,
        created_at: created_at,
        reblogs_count: announcement_count,
-       replies_count: 0,
 -      replies_count: object["repliesCount"] || 0,
++      replies_count: object.data["repliesCount"] || 0,
        favourites_count: like_count,
-       reblogged: !!repeated,
-       favourited: !!favorited,
-       muted: false,
+       reblogged: reblogged?(activity, opts[:for]),
+       favourited: present?(favorited),
+       bookmarked: present?(bookmarked),
+       muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
+       pinned: pinned?(activity, user),
        sensitive: sensitive,
-       spoiler_text: object.data["summary"] || "",
-       visibility: get_visibility(object.data),
-       media_attachments: attachments |> Enum.take(4),
+       spoiler_text: summary_html,
+       visibility: get_visibility(object),
+       media_attachments: attachments,
        mentions: mentions,
-       # fix,
-       tags: [],
+       tags: build_tags(tags),
        application: %{
          name: "Web",
          website: nil
        },
        language: nil,
-       emojis: emojis
 -      emojis: build_emojis(activity.data["object"]["emoji"]),
++      emojis: build_emojis(object.data["emoji"]),
+       pleroma: %{
+         local: activity.local,
+         conversation_id: get_context_id(activity),
+         content: %{"text/plain" => content_plaintext},
+         spoiler_text: %{"text/plain" => summary_plaintext}
+       }
      }
    end
  
@@@ -194,15 -305,15 +312,19 @@@
    end
  
    def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
 -    with nil <- replied_to_activities[activity.data["object"]["inReplyTo"]] do
 +    object = Object.normalize(activity.data["object"])
-     replied_to_activities[object.data["inReplyTo"]]
++
++    with nil <- replied_to_activities[object.data["inReplyTo"]] do
+       # If user didn't participate in the thread
+       Activity.get_in_reply_to_activity(activity)
+     end
    end
  
    def get_reply_to(%{data: %{"object" => object}}, _) do
 -    if object["inReplyTo"] && object["inReplyTo"] != "" do
 -      Activity.get_create_by_object_ap_id(object["inReplyTo"])
 +    object = Object.normalize(object)
 +
 +    if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do
-       Activity.get_create_activity_by_object_ap_id(object.data["inReplyTo"])
++      Activity.get_create_by_object_ap_id(object.data["inReplyTo"])
      else
        nil
      end
@@@ -210,8 -321,8 +332,8 @@@
  
    def get_visibility(object) do
      public = "https://www.w3.org/ns/activitystreams#Public"
--    to = object["to"] || []
--    cc = object["cc"] || []
++    to = object.data["to"] || []
++    cc = object.data["cc"] || []
  
      cond do
        public in to ->
@@@ -229,31 -343,81 +354,81 @@@
      end
    end
  
--  def render_content(%{"type" => "Video"} = object) do
-     name = object["name"]
 -    with name when not is_nil(name) and name != "" <- object["name"] do
 -      "<p><a href=\"#{object["id"]}\">#{name}</a></p>#{object["content"]}"
++  def render_content(%{data: %{"type" => "Video"}} = object) do
++    with name when not is_nil(name) and name != "" <- object.data["name"] do
++      "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"
+     else
 -      _ -> object["content"] || ""
++      _ -> object.data["content"] || ""
+     end
+   end
  
-     content =
-       if !!name and name != "" do
-         "<p><a href=\"#{object["id"]}\">#{name}</a></p>#{object["content"]}"
-       else
-         object["content"] || ""
-       end
 -  def render_content(%{"type" => object_type} = object)
++  def render_content(%{data: %{"type" => object_type}} = object)
+       when object_type in ["Article", "Page"] do
 -    with summary when not is_nil(summary) and summary != "" <- object["name"],
 -         url when is_bitstring(url) <- object["url"] do
 -      "<p><a href=\"#{url}\">#{summary}</a></p>#{object["content"]}"
++    with summary when not is_nil(summary) and summary != "" <- object.data["name"],
++         url when is_bitstring(url) <- object.data["url"] do
++      "<p><a href=\"#{url}\">#{summary}</a></p>#{object.data["content"]}"
+     else
 -      _ -> object["content"] || ""
++      _ -> object.data["content"] || ""
+     end
+   end
  
-     content
 -  def render_content(object), do: object["content"] || ""
++  def render_content(object), do: object.data["content"] || ""
+ 
+   @doc """
+   Builds a dictionary tags.
+ 
+   ## Examples
+ 
+   iex> Pleroma.Web.MastodonAPI.StatusView.build_tags(["fediverse", "nextcloud"])
+   [{"name": "fediverse", "url": "/tag/fediverse"},
+    {"name": "nextcloud", "url": "/tag/nextcloud"}]
+ 
+   """
+   @spec build_tags(list(any())) :: list(map())
+   def build_tags(object_tags) when is_list(object_tags) do
+     object_tags = for tag when is_binary(tag) <- object_tags, do: tag
+ 
+     Enum.reduce(object_tags, [], fn tag, tags ->
+       tags ++ [%{name: tag, url: "/tag/#{tag}"}]
+     end)
    end
  
-   def render_content(%{"type" => object_type} = object) when object_type in ["Article", "Page"] do
-     summary = object["name"]
+   def build_tags(_), do: []
  
-     content =
-       if !!summary and summary != "" and is_bitstring(object["url"]) do
-         "<p><a href=\"#{object["url"]}\">#{summary}</a></p>#{object["content"]}"
-       else
-         object["content"] || ""
-       end
+   @doc """
+   Builds list emojis.
  
-     content
+   Arguments: `nil` or list tuple of name and url.
+ 
+   Returns list emojis.
+ 
+   ## Examples
+ 
+   iex> Pleroma.Web.MastodonAPI.StatusView.build_emojis([{"2hu", "corndog.png"}])
+   [%{shortcode: "2hu", static_url: "corndog.png", url: "corndog.png", visible_in_picker: false}]
+ 
+   """
+   @spec build_emojis(nil | list(tuple())) :: list(map())
+   def build_emojis(nil), do: []
+ 
+   def build_emojis(emojis) do
+     emojis
+     |> Enum.map(fn {name, url} ->
+       name = HTML.strip_tags(name)
+ 
+       url =
+         url
+         |> HTML.strip_tags()
+         |> MediaProxy.url()
+ 
+       %{shortcode: name, url: url, static_url: url, visible_in_picker: false}
+     end)
    end
  
-   def render_content(object), do: object["content"] || ""
+   defp present?(nil), do: false
+   defp present?(false), do: false
+   defp present?(_), do: true
+ 
+   defp pinned?(%Activity{id: id}, %User{info: %{pinned_activities: pinned_activities}}),
+     do: id in pinned_activities
  end
diff --cc lib/pleroma/web/ostatus/handlers/note_handler.ex
index ba232b0ec,db995ec77..ec6e5cfaf
--- a/lib/pleroma/web/ostatus/handlers/note_handler.ex
+++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex
@@@ -104,13 -111,12 +111,13 @@@ defmodule Pleroma.Web.OStatus.NoteHandl
           {:ok, actor} <- OStatus.find_make_or_update_user(author),
           content_html <- OStatus.get_content(entry),
           cw <- OStatus.get_cw(entry),
-          inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
-          inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo),
-          inReplyToObject <-
-            (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil,
-          inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo,
+          in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
+          in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to),
 -         in_reply_to <-
 -           (in_reply_to_activity && in_reply_to_activity.data["object"]["id"]) || in_reply_to,
++         in_reply_to_object <-
++           (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil,
++         in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to,
           attachments <- OStatus.get_attachments(entry),
-          context <- get_context(entry, inReplyTo),
+          context <- get_context(entry, in_reply_to),
           tags <- OStatus.get_tags(entry),
           mentions <- get_mentions(entry),
           to <- make_to_list(actor, mentions),
diff --cc lib/pleroma/web/twitter_api/controllers/util_controller.ex
index b0ed8387e,ed45ca735..9441984c7
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@@ -64,23 -74,39 +74,39 @@@ defmodule Pleroma.Web.TwitterAPI.UtilCo
    end
  
    def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
-     {err, followee} = OStatus.find_or_make_user(acct)
-     avatar = User.avatar_url(followee)
-     name = followee.nickname
-     id = followee.id
- 
-     if !!user do
-       conn
-       |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id})
+     if is_status?(acct) do
 -      {:ok, object} = ActivityPub.fetch_object_from_id(acct)
++      {:ok, object} = Pleroma.Object.Fetcher.fetch_object_from_id(acct)
+       %Activity{id: activity_id} = Activity.get_create_by_object_ap_id(object.data["id"])
+       redirect(conn, to: "/notice/#{activity_id}")
      else
-       conn
-       |> render("follow_login.html", %{
-         error: false,
-         acct: acct,
-         avatar: avatar,
-         name: name,
-         id: id
-       })
+       {err, followee} = OStatus.find_or_make_user(acct)
+       avatar = User.avatar_url(followee)
+       name = followee.nickname
+       id = followee.id
+ 
+       if !!user do
+         conn
+         |> render("follow.html", %{error: err, acct: acct, avatar: avatar, name: name, id: id})
+       else
+         conn
+         |> render("follow_login.html", %{
+           error: false,
+           acct: acct,
+           avatar: avatar,
+           name: name,
+           id: id
+         })
+       end
+     end
+   end
+ 
+   defp is_status?(acct) do
 -    case ActivityPub.fetch_and_contain_remote_object_from_id(acct) do
++    case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do
+       {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] ->
+         true
+ 
+       _ ->
+         false
      end
    end
  
diff --cc lib/pleroma/web/twitter_api/views/activity_view.ex
index 18b2ebb0b,ecb2b437b..c64152da8
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@@ -211,16 -228,17 +228,19 @@@ defmodule Pleroma.Web.TwitterAPI.Activi
        ) do
      user = get_user(activity.data["actor"], opts)
  
 -    created_at = object["published"] |> Utils.date_to_asctime()
 -    like_count = object["like_count"] || 0
 -    announcement_count = object["announcement_count"] || 0
 -    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
 -    repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
 +    object = Object.normalize(object_id)
 +
 +    created_at = object.data["published"] |> Utils.date_to_asctime()
 +    like_count = object.data["like_count"] || 0
 +    announcement_count = object.data["announcement_count"] || 0
 +    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 +    repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || [])
+     pinned = activity.id in user.info.pinned_activities
  
      attentions =
-       activity.recipients
+       []
+       |> Utils.maybe_notify_to_recipients(activity)
+       |> Utils.maybe_notify_mentioned_recipients(activity)
        |> Enum.map(fn ap_id -> get_user(ap_id, opts) end)
        |> Enum.filter(& &1)
        |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end)
@@@ -232,22 -250,44 +252,44 @@@
  
      tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
  
 -    {summary, content} = render_content(object)
 +    {summary, content} = render_content(object.data)
  
      html =
-       HTML.filter_tags(content, User.html_filter_policy(opts[:for]))
+       content
+       |> HTML.get_cached_scrubbed_html_for_activity(
+         User.html_filter_policy(opts[:for]),
+         activity,
+         "twitterapi:content"
+       )
 -      |> Formatter.emojify(object["emoji"])
 +      |> Formatter.emojify(object.data["emoji"])
  
+     text =
+       if content do
+         content
+         |> String.replace(~r/<br\s?\/?>/, "\n")
+         |> HTML.get_cached_stripped_html_for_activity(activity, "twitterapi:content")
+       else
+         ""
+       end
+ 
      reply_parent = Activity.get_in_reply_to_activity(activity)
  
      reply_user = reply_parent && User.get_cached_by_ap_id(reply_parent.actor)
  
+     summary = HTML.strip_tags(summary)
+ 
+     card =
+       StatusView.render(
+         "card.json",
+         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+       )
+ 
      %{
        "id" => activity.id,
 -      "uri" => activity.data["object"]["id"],
 +      "uri" => object.data["id"],
        "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
        "statusnet_html" => html,
-       "text" => HTML.strip_tags(content),
+       "text" => text,
        "is_local" => activity.local,
        "is_post_verb" => true,
        "created_at" => created_at,
@@@ -263,12 -303,16 +305,16 @@@
        "repeat_num" => announcement_count,
        "favorited" => !!favorited,
        "repeated" => !!repeated,
+       "pinned" => pinned,
 -      "external_url" => object["external_url"] || object["id"],
 +      "external_url" => object.data["external_url"] || object.data["id"],
        "tags" => tags,
        "activity_type" => "post",
        "possibly_sensitive" => possibly_sensitive,
-       "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data),
-       "summary" => summary
+       "visibility" => StatusView.get_visibility(object),
+       "summary" => summary,
 -      "summary_html" => summary |> Formatter.emojify(object["emoji"]),
++      "summary_html" => summary |> Formatter.emojify(object.data["emoji"]),
+       "card" => card,
+       "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user)
      }
    end
  
diff --cc test/object_test.exs
index dac6c3be7,911757d57..a30efd48c
--- a/test/object_test.exs
+++ b/test/object_test.exs
@@@ -47,28 -54,8 +54,30 @@@ defmodule Pleroma.ObjectTest d
        cached_object = Object.get_cached_by_ap_id(object.data["id"])
  
        refute object == cached_object
+ 
+       assert cached_object.data["type"] == "Tombstone"
      end
    end
 +
 +  describe "normalizer" do
 +    test "fetches unknown objects by default" do
 +      %Object{} =
 +        object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367")
 +
 +      assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
 +    end
 +
 +    test "fetches unknown objects when fetch_remote is explicitly true" do
 +      %Object{} =
 +        object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true)
 +
 +      assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
 +    end
 +
 +    test "does not fetch unknown objects when fetch_remote is false" do
 +      assert is_nil(
 +               Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)
 +             )
 +    end
 +  end
  end
diff --cc test/web/activity_pub/activity_pub_test.exs
index bc9fcc75d,17fec05b1..68bfb3858
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@@@ -362,16 -635,43 +641,53 @@@ defmodule Pleroma.Web.ActivityPub.Activ
      end
    end
  
 +  describe "fetch the latest Follow" do
 +    test "fetches the latest Follow activity" do
 +      %Activity{data: %{"type" => "Follow"}} = activity = insert(:follow_activity)
 +      follower = Repo.get_by(User, ap_id: activity.data["actor"])
 +      followed = Repo.get_by(User, ap_id: activity.data["object"])
 +
 +      assert activity == Utils.fetch_latest_follow(follower, followed)
 +    end
 +  end
 +
+   describe "fetching an object" do
+     test "it fetches an object" do
+       {:ok, object} =
+         ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+ 
+       assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
+       assert activity.data["id"]
+ 
+       {:ok, object_again} =
+         ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+ 
+       assert [attachment] = object.data["attachment"]
+       assert is_list(attachment["url"])
+ 
+       assert object == object_again
+     end
+ 
+     test "it works with objects only available via Ostatus" do
+       {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
+       assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
+       assert activity.data["id"]
+ 
+       {:ok, object_again} =
+         ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
+ 
+       assert object == object_again
+     end
+ 
+     test "it correctly stitches up conversations between ostatus and ap" do
+       last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
+       {:ok, object} = ActivityPub.fetch_object_from_id(last)
+ 
+       object = Object.get_by_ap_id(object.data["inReplyTo"])
+       assert object
+     end
+   end
+ 
    describe "following / unfollowing" do
      test "creates a follow activity" do
        follower = insert(:user)
diff --cc test/web/activity_pub/transmogrifier_test.exs
index ea9d9fe58,c857a7ec1..5559cdf87
--- a/test/web/activity_pub/transmogrifier_test.exs
+++ b/test/web/activity_pub/transmogrifier_test.exs
@@@ -40,16 -50,14 +50,14 @@@ defmodule Pleroma.Web.ActivityPub.Trans
          |> Map.put("object", object)
  
        {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
 +      returned_object = Object.normalize(returned_activity.data["object"])
  
        assert activity =
-                Activity.get_create_activity_by_object_ap_id(
+                Activity.get_create_by_object_ap_id(
                   "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
                 )
  
 -      assert returned_activity.data["object"]["inReplyToAtomUri"] ==
 -               "https://shitposter.club/notice/2827873"
 +      assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
- 
-       assert returned_object.data["inReplyToStatusId"] == activity.id
      end
  
      test "it works for incoming notices" do
@@@ -155,11 -157,40 +163,41 @@@
        data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!()
  
        {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
 +      object = Object.normalize(data["object"])
  
 -      assert data["object"]["url"] == "https://prismo.news/posts/83"
 +      assert object.data["url"] == "https://prismo.news/posts/83"
      end
  
+     test "it cleans up incoming notices which are not really DMs" do
+       user = insert(:user)
+       other_user = insert(:user)
+ 
+       to = [user.ap_id, other_user.ap_id]
+ 
+       data =
+         File.read!("test/fixtures/mastodon-post-activity.json")
+         |> Poison.decode!()
+         |> Map.put("to", to)
+         |> Map.put("cc", [])
+ 
+       object =
+         data["object"]
+         |> Map.put("to", to)
+         |> Map.put("cc", [])
+ 
+       data = Map.put(data, "object", object)
+ 
+       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+ 
+       assert data["to"] == []
+       assert data["cc"] == to
+ 
+       object = data["object"]
+ 
+       assert object["to"] == []
+       assert object["cc"] == to
+     end
+ 
      test "it works for incoming follow requests" do
        user = insert(:user)
  
@@@ -281,9 -312,72 +319,72 @@@
        assert data["id"] ==
                 "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity"
  
 -      assert data["object"] == activity.data["object"]["id"]
 +      assert data["object"] == activity.data["object"]
  
-       assert Activity.get_create_activity_by_object_ap_id(data["object"]).id == activity.id
+       assert Activity.get_create_by_object_ap_id(data["object"]).id == activity.id
+     end
+ 
+     test "it does not clobber the addressing on announce activities" do
+       user = insert(:user)
+       {:ok, activity} = CommonAPI.post(user, %{"status" => "hey"})
+ 
+       data =
+         File.read!("test/fixtures/mastodon-announce.json")
+         |> Poison.decode!()
+         |> Map.put("object", activity.data["object"]["id"])
+         |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"])
+         |> Map.put("cc", [])
+ 
+       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+ 
+       assert data["to"] == ["http://mastodon.example.org/users/admin/followers"]
+     end
+ 
+     test "it ensures that as:Public activities make it to their followers collection" do
+       user = insert(:user)
+ 
+       data =
+         File.read!("test/fixtures/mastodon-post-activity.json")
+         |> Poison.decode!()
+         |> Map.put("actor", user.ap_id)
+         |> Map.put("to", ["https://www.w3.org/ns/activitystreams#Public"])
+         |> Map.put("cc", [])
+ 
+       object =
+         data["object"]
+         |> Map.put("attributedTo", user.ap_id)
+         |> Map.put("to", ["https://www.w3.org/ns/activitystreams#Public"])
+         |> Map.put("cc", [])
+ 
+       data = Map.put(data, "object", object)
+ 
+       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+ 
+       assert data["cc"] == [User.ap_followers(user)]
+     end
+ 
+     test "it ensures that address fields become lists" do
+       user = insert(:user)
+ 
+       data =
+         File.read!("test/fixtures/mastodon-post-activity.json")
+         |> Poison.decode!()
+         |> Map.put("actor", user.ap_id)
+         |> Map.put("to", nil)
+         |> Map.put("cc", nil)
+ 
+       object =
+         data["object"]
+         |> Map.put("attributedTo", user.ap_id)
+         |> Map.put("to", nil)
+         |> Map.put("cc", nil)
+ 
+       data = Map.put(data, "object", object)
+ 
+       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+ 
+       assert !is_nil(data["to"])
+       assert !is_nil(data["cc"])
      end
  
      test "it works for incoming update activities" do
diff --cc test/web/common_api/common_api_test.exs
index 3dc5f6f84,34aa5bf18..b9ed088e4
--- a/test/web/common_api/common_api_test.exs
+++ b/test/web/common_api/common_api_test.exs
@@@ -1,7 -1,12 +1,13 @@@
- defmodule Pleroma.Web.CommonAPI.Test do
+ # Pleroma: A lightweight social networking server
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+ # SPDX-License-Identifier: AGPL-3.0-only
+ 
+ defmodule Pleroma.Web.CommonAPITest do
    use Pleroma.DataCase
+   alias Pleroma.Activity
+   alias Pleroma.User
++  alias Pleroma.Object
    alias Pleroma.Web.CommonAPI
-   alias Pleroma.{User, Object}
  
    import Pleroma.Factory
  
@@@ -9,11 -32,16 +33,18 @@@
      user = insert(:user)
      {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"})
  
 -    assert activity.data["object"]["tag"] == ["2hu"]
 +    object = Object.normalize(activity.data["object"])
 +
 +    assert object.data["tag"] == ["2hu"]
    end
  
+   test "it adds emoji in the object" do
+     user = insert(:user)
+     {:ok, activity} = CommonAPI.post(user, %{"status" => ":moominmamma:"})
+ 
+     assert activity.data["object"]["emoji"]["moominmamma"]
+   end
+ 
    test "it adds emoji when updating profiles" do
      user = insert(:user, %{name: ":karjalanpiirakka:"})
  
@@@ -52,9 -79,187 +83,188 @@@
            "content_type" => "text/markdown"
          })
  
 -      content = activity.data["object"]["content"]
 -      assert content == "<p><b>2hu</b></p>alert('xss')"
 +      object = Object.normalize(activity.data["object"])
 +
 +      assert object.data["content"] == "<p><b>2hu</b></p>alert('xss')"
      end
    end
+ 
+   describe "reactions" do
+     test "repeating a status" do
+       user = insert(:user)
+       other_user = insert(:user)
+ 
+       {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"})
+ 
+       {:ok, %Activity{}, _} = CommonAPI.repeat(activity.id, user)
+     end
+ 
+     test "favoriting a status" do
+       user = insert(:user)
+       other_user = insert(:user)
+ 
+       {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"})
+ 
+       {:ok, %Activity{}, _} = CommonAPI.favorite(activity.id, user)
+     end
+ 
+     test "retweeting a status twice returns an error" do
+       user = insert(:user)
+       other_user = insert(:user)
+ 
+       {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"})
+       {:ok, %Activity{}, _object} = CommonAPI.repeat(activity.id, user)
+       {:error, _} = CommonAPI.repeat(activity.id, user)
+     end
+ 
+     test "favoriting a status twice returns an error" do
+       user = insert(:user)
+       other_user = insert(:user)
+ 
+       {:ok, activity} = CommonAPI.post(other_user, %{"status" => "cofe"})
+       {:ok, %Activity{}, _object} = CommonAPI.favorite(activity.id, user)
+       {:error, _} = CommonAPI.favorite(activity.id, user)
+     end
+   end
+ 
+   describe "pinned statuses" do
+     setup do
+       Pleroma.Config.put([:instance, :max_pinned_statuses], 1)
+ 
+       user = insert(:user)
+       {:ok, activity} = CommonAPI.post(user, %{"status" => "HI!!!"})
+ 
+       [user: user, activity: activity]
+     end
+ 
+     test "pin status", %{user: user, activity: activity} do
+       assert {:ok, ^activity} = CommonAPI.pin(activity.id, user)
+ 
+       id = activity.id
+       user = refresh_record(user)
+ 
+       assert %User{info: %{pinned_activities: [^id]}} = user
+     end
+ 
+     test "only self-authored can be pinned", %{activity: activity} do
+       user = insert(:user)
+ 
+       assert {:error, "Could not pin"} = CommonAPI.pin(activity.id, user)
+     end
+ 
+     test "max pinned statuses", %{user: user, activity: activity_one} do
+       {:ok, activity_two} = CommonAPI.post(user, %{"status" => "HI!!!"})
+ 
+       assert {:ok, ^activity_one} = CommonAPI.pin(activity_one.id, user)
+ 
+       user = refresh_record(user)
+ 
+       assert {:error, "You have already pinned the maximum number of statuses"} =
+                CommonAPI.pin(activity_two.id, user)
+     end
+ 
+     test "unpin status", %{user: user, activity: activity} do
+       {:ok, activity} = CommonAPI.pin(activity.id, user)
+ 
+       user = refresh_record(user)
+ 
+       assert {:ok, ^activity} = CommonAPI.unpin(activity.id, user)
+ 
+       user = refresh_record(user)
+ 
+       assert %User{info: %{pinned_activities: []}} = user
+     end
+ 
+     test "should unpin when deleting a status", %{user: user, activity: activity} do
+       {:ok, activity} = CommonAPI.pin(activity.id, user)
+ 
+       user = refresh_record(user)
+ 
+       assert {:ok, _} = CommonAPI.delete(activity.id, user)
+ 
+       user = refresh_record(user)
+ 
+       assert %User{info: %{pinned_activities: []}} = user
+     end
+   end
+ 
+   describe "mute tests" do
+     setup do
+       user = insert(:user)
+ 
+       activity = insert(:note_activity)
+ 
+       [user: user, activity: activity]
+     end
+ 
+     test "add mute", %{user: user, activity: activity} do
+       {:ok, _} = CommonAPI.add_mute(user, activity)
+       assert CommonAPI.thread_muted?(user, activity)
+     end
+ 
+     test "remove mute", %{user: user, activity: activity} do
+       CommonAPI.add_mute(user, activity)
+       {:ok, _} = CommonAPI.remove_mute(user, activity)
+       refute CommonAPI.thread_muted?(user, activity)
+     end
+ 
+     test "check that mutes can't be duplicate", %{user: user, activity: activity} do
+       CommonAPI.add_mute(user, activity)
+       {:error, _} = CommonAPI.add_mute(user, activity)
+     end
+   end
+ 
+   describe "reports" do
+     test "creates a report" do
+       reporter = insert(:user)
+       target_user = insert(:user)
+ 
+       {:ok, activity} = CommonAPI.post(target_user, %{"status" => "foobar"})
+ 
+       reporter_ap_id = reporter.ap_id
+       target_ap_id = target_user.ap_id
+       activity_ap_id = activity.data["id"]
+       comment = "foobar"
+ 
+       report_data = %{
+         "account_id" => target_user.id,
+         "comment" => comment,
+         "status_ids" => [activity.id]
+       }
+ 
+       assert {:ok, flag_activity} = CommonAPI.report(reporter, report_data)
+ 
+       assert %Activity{
+                actor: ^reporter_ap_id,
+                data: %{
+                  "type" => "Flag",
+                  "content" => ^comment,
+                  "object" => [^target_ap_id, ^activity_ap_id]
+                }
+              } = flag_activity
+     end
+   end
+ 
+   describe "reblog muting" do
+     setup do
+       muter = insert(:user)
+ 
+       muted = insert(:user)
+ 
+       [muter: muter, muted: muted]
+     end
+ 
+     test "add a reblog mute", %{muter: muter, muted: muted} do
+       {:ok, muter} = CommonAPI.hide_reblogs(muter, muted)
+ 
+       assert Pleroma.User.showing_reblogs?(muter, muted) == false
+     end
+ 
+     test "remove a reblog mute", %{muter: muter, muted: muted} do
+       {:ok, muter} = CommonAPI.hide_reblogs(muter, muted)
+       {:ok, muter} = CommonAPI.show_reblogs(muter, muted)
+ 
+       assert Pleroma.User.showing_reblogs?(muter, muted) == true
+     end
+   end
  end
diff --cc test/web/mastodon_api/status_view_test.exs
index 4f58ce8af,db2fdc2f6..4ea50c7c6
--- a/test/web/mastodon_api/status_view_test.exs
+++ b/test/web/mastodon_api/status_view_test.exs
@@@ -1,24 -5,64 +5,66 @@@
  defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
    use Pleroma.DataCase
  
-   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView}
-   alias Pleroma.{Repo, User, Object}
-   alias Pleroma.Web.OStatus
+   alias Pleroma.Activity
+   alias Pleroma.User
++  alias Pleroma.Repo
++  alias Pleroma.Object
+   alias Pleroma.Web.ActivityPub.ActivityPub
    alias Pleroma.Web.CommonAPI
+   alias Pleroma.Web.CommonAPI.Utils
+   alias Pleroma.Web.MastodonAPI.AccountView
+   alias Pleroma.Web.MastodonAPI.StatusView
+   alias Pleroma.Web.OStatus
    import Pleroma.Factory
+   import Tesla.Mock
+ 
+   setup do
+     mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+     :ok
+   end
+ 
+   test "returns a temporary ap_id based user for activities missing db users" do
+     user = insert(:user)
+ 
+     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"})
+ 
+     Repo.delete(user)
+     Cachex.clear(:user_cache)
+ 
+     %{account: ms_user} = StatusView.render("status.json", activity: activity)
+ 
+     assert ms_user.acct == "erroruser@example.com"
+   end
+ 
+   test "tries to get a user by nickname if fetching by ap_id doesn't work" do
+     user = insert(:user)
+ 
+     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"})
+ 
+     {:ok, user} =
+       user
+       |> Ecto.Changeset.change(%{ap_id: "#{user.ap_id}/extension/#{user.nickname}"})
+       |> Repo.update()
+ 
+     Cachex.clear(:user_cache)
+ 
+     result = StatusView.render("status.json", activity: activity)
+ 
+     assert result[:account][:id] == to_string(user.id)
+   end
  
    test "a note with null content" do
      note = insert(:note_activity)
 +    note_object = Object.normalize(note.data["object"])
  
      data =
 -      note.data
 -      |> put_in(["object", "content"], nil)
 +      note_object.data
 +      |> Map.put("content", nil)
  
 -    note =
 -      note
 -      |> Map.put(:data, data)
 +    Object.change(note_object, %{data: data})
 +    |> Repo.update()
  
-     user = User.get_cached_by_ap_id(note.data["actor"])
+     User.get_cached_by_ap_id(note.data["actor"])
  
      status = StatusView.render("status.json", %{activity: note})
  
diff --cc test/web/ostatus/ostatus_test.exs
index b5805c668,9fd100f63..50467c71f
--- a/test/web/ostatus/ostatus_test.exs
+++ b/test/web/ostatus/ostatus_test.exs
@@@ -150,9 -154,7 +163,8 @@@ defmodule Pleroma.Web.OStatusTest d
      assert "https://pleroma.soykaf.com/users/lain" in activity.data["to"]
      refute activity.local
  
-     retweeted_activity = Repo.get(Activity, retweeted_activity.id)
+     retweeted_activity = Activity.get_by_id(retweeted_activity.id)
 +    retweeted_object = Object.normalize(retweeted_activity.data["object"])
- 
      assert retweeted_activity.data["type"] == "Create"
      assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain"
      refute retweeted_activity.local
diff --cc test/web/twitter_api/twitter_api_test.exs
index bc53fe68a,4c9ae2da8..5bea1037a
--- a/test/web/twitter_api/twitter_api_test.exs
+++ b/test/web/twitter_api/twitter_api_test.exs
@@@ -33,14 -46,13 +46,14 @@@ defmodule Pleroma.Web.TwitterAPI.Twitte
      }
  
      {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input)
 +    object = Object.normalize(activity.data["object"])
  
      expected_text =
-       "Hello again, <span><a class='mention' href='shp'>@<span>shp</span></a></span>.&lt;script&gt;&lt;/script&gt;<br>This is on another :moominmamma: line. <a href='http://localhost:4001/tag/2hu' rel='tag'>#2hu</a> <a href='http://localhost:4001/tag/epic' rel='tag'>#epic</a> <a href='http://localhost:4001/tag/phantasmagoric' rel='tag'>#phantasmagoric</a><br><a href=\"http://example.org/image.jpg\" class='attachment'>image.jpg</a>"
+       "Hello again, <span class='h-card'><a data-user='#{mentioned_user.id}' class='u-url mention' href='shp'>@<span>shp</span></a></span>.&lt;script&gt;&lt;/script&gt;<br>This is on another :moominmamma: line. <a class='hashtag' data-tag='2hu' href='http://localhost:4001/tag/2hu' rel='tag'>#2hu</a> <a class='hashtag' data-tag='epic' href='http://localhost:4001/tag/epic' rel='tag'>#epic</a> <a class='hashtag' data-tag='phantasmagoric' href='http://localhost:4001/tag/phantasmagoric' rel='tag'>#phantasmagoric</a><br><a href=\"http://example.org/image.jpg\" class='attachment'>image.jpg</a>"
  
 -    assert get_in(activity.data, ["object", "content"]) == expected_text
 -    assert get_in(activity.data, ["object", "type"]) == "Note"
 -    assert get_in(activity.data, ["object", "actor"]) == user.ap_id
 +    assert get_in(object.data, ["content"]) == expected_text
 +    assert get_in(object.data, ["type"]) == "Note"
 +    assert get_in(object.data, ["actor"]) == user.ap_id
      assert get_in(activity.data, ["actor"]) == user.ap_id
      assert Enum.member?(get_in(activity.data, ["cc"]), User.ap_followers(user))
  
@@@ -91,10 -101,11 +104,10 @@@
  
      assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"])
  
 -    assert get_in(reply.data, ["object", "context"]) ==
 -             get_in(activity.data, ["object", "context"])
 +    assert get_in(reply_object.data, ["context"]) == get_in(object.data, ["context"])
  
 -    assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"])
 +    assert get_in(reply_object.data, ["inReplyTo"]) == get_in(activity.data, ["object"])
-     assert get_in(reply_object.data, ["inReplyToStatusId"]) == activity.id
+     assert Activity.get_in_reply_to_activity(reply).id == activity.id
    end
  
    test "Follow another user using user_id" do
diff --cc test/web/twitter_api/views/activity_view_test.exs
index f4741cf24,ee9a0c834..7ef0270cc
--- a/test/web/twitter_api/views/activity_view_test.exs
+++ b/test/web/twitter_api/views/activity_view_test.exs
@@@ -1,6 -5,10 +5,11 @@@
  defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
    use Pleroma.DataCase
  
+   alias Pleroma.Activity
++  alias Pleroma.Object
+   alias Pleroma.Repo
+   alias Pleroma.User
+   alias Pleroma.Web.ActivityPub.ActivityPub
    alias Pleroma.Web.CommonAPI
    alias Pleroma.Web.CommonAPI.Utils
    alias Pleroma.Web.TwitterAPI.ActivityView
@@@ -23,7 -128,7 +130,7 @@@
  
      result = ActivityView.render("activity.json", activity: activity)
  
-     convo_id = TwitterAPI.context_to_conversation_id(object.data["context"])
 -    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
++    convo_id = Utils.context_to_conversation_id(object.data["context"])
  
      expected = %{
        "activity_type" => "post",
@@@ -46,15 -151,21 +153,21 @@@
        "possibly_sensitive" => false,
        "repeat_num" => 0,
        "repeated" => false,
+       "pinned" => false,
        "statusnet_conversation_id" => convo_id,
+       "summary" => "",
+       "summary_html" => "",
        "statusnet_html" =>
-         "Hey <span><a href=\"#{other_user.ap_id}\">@<span>shp</span></a></span>!",
+         "Hey <span class=\"h-card\"><a data-user=\"#{other_user.id}\" class=\"u-url mention\" href=\"#{
+           other_user.ap_id
+         }\">@<span>shp</span></a></span>!",
        "tags" => [],
        "text" => "Hey @shp!",
 -      "uri" => activity.data["object"]["id"],
 +      "uri" => object.data["id"],
        "user" => UserView.render("show.json", %{user: user}),
        "visibility" => "direct",
-       "summary" => nil
+       "card" => nil,
+       "muted" => false
      }
  
      assert result == expected
@@@ -64,9 -175,8 +177,9 @@@
      user = insert(:user)
      other_user = insert(:user, %{nickname: "shp"})
      {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"})
 +    object = Object.normalize(activity.data["object"])
  
-     convo_id = TwitterAPI.context_to_conversation_id(object.data["context"])
 -    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
++    convo_id = Utils.context_to_conversation_id(object.data["context"])
  
      mocks = [
        {
@@@ -164,11 -277,11 +280,11 @@@
      other_user = insert(:user, %{nickname: "shp"})
  
      {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"})
 -    {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user)
 +    {:ok, announce, object} = CommonAPI.repeat(activity.id, other_user)
  
-     convo_id = TwitterAPI.context_to_conversation_id(object.data["context"])
 -    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
++    convo_id = Utils.context_to_conversation_id(object.data["context"])
  
-     activity = Repo.get(Activity, activity.id)
+     activity = Activity.get_by_id(activity.id)
  
      result = ActivityView.render("activity.json", activity: announce)