Merge branch 'develop' into test/activity_pub/transmogrifier.ex
authorMaksim Pechnikov <parallel588@gmail.com>
Thu, 19 Sep 2019 04:35:34 +0000 (07:35 +0300)
committerMaksim Pechnikov <parallel588@gmail.com>
Thu, 19 Sep 2019 04:35:34 +0000 (07:35 +0300)
1  2 
lib/pleroma/object/fetcher.ex
lib/pleroma/web/activity_pub/transmogrifier.ex
lib/pleroma/web/activity_pub/views/user_view.ex
test/web/activity_pub/transmogrifier_test.exs
test/web/activity_pub/views/user_view_test.exs

index 2217d1eb302546ad937e3b6994746063cc80e9d0,cea33b5af26e9ef3a56b12f688ca63a6de0c5a52..5e064fd8735a9ea9dbb48f0e13d872204db1b2b4
@@@ -6,19 -6,39 +6,40 @@@ defmodule Pleroma.Object.Fetcher d
    alias Pleroma.HTTP
    alias Pleroma.Object
    alias Pleroma.Object.Containment
+   alias Pleroma.Repo
    alias Pleroma.Signature
    alias Pleroma.Web.ActivityPub.InternalFetchActor
    alias Pleroma.Web.ActivityPub.Transmogrifier
    alias Pleroma.Web.OStatus
  
    require Logger
+   require Pleroma.Constants
  
-   @spec reinject_object(map()) :: {:ok, Object.t()} | {:error, any()}
-   defp reinject_object(data) do
+   defp touch_changeset(changeset) do
+     updated_at =
+       NaiveDateTime.utc_now()
+       |> NaiveDateTime.truncate(:second)
+     Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
+   end
+   defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+     internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
+     Map.merge(data, internal_fields)
+   end
+   defp maybe_reinject_internal_fields(data, _), do: data
++  @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
+   defp reinject_object(struct, data) do
      Logger.debug("Reinjecting object #{data["id"]}")
  
      with data <- Transmogrifier.fix_object(data),
-          {:ok, object} <- Object.create(data) do
+          data <- maybe_reinject_internal_fields(data, struct),
+          changeset <- Object.change(struct, %{data: data}),
+          changeset <- touch_changeset(changeset),
+          {:ok, object} <- Repo.insert_or_update(changeset) do
        {:ok, object}
      else
        e ->
      end
    end
  
+   def refetch_object(%Object{data: %{"id" => id}} = object) do
+     with {:local, false} <- {:local, String.starts_with?(id, Pleroma.Web.base_url() <> "/")},
+          {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+          {:ok, object} <- reinject_object(object, data) do
+       {:ok, object}
+     else
+       {:local, true} -> object
+       e -> {:error, e}
+     end
+   end
    # TODO:
    # This will create a Create activity, which we need internally at the moment.
    def fetch_object_from_id(id, options \\ []) do
 -    if object = Object.get_cached_by_ap_id(id) do
 +    with {:fetch_object, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
 +         {:fetch, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
 +         {:normalize, nil} <- {:normalize, Object.normalize(data, false)},
 +         params <- prepare_activity_params(data),
 +         {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)},
 +         {:ok, activity} <- Transmogrifier.handle_incoming(params, options),
 +         {:object, _data, %Object{} = object} <-
 +           {:object, data, Object.normalize(activity, false)} do
        {:ok, object}
      else
 -      Logger.info("Fetching #{id} via AP")
 -
 -      with {:fetch, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
 -           {:normalize, nil} <- {:normalize, Object.normalize(data, false)},
 -           params <- %{
 -             "type" => "Create",
 -             "to" => data["to"],
 -             "cc" => data["cc"],
 -             # Should we seriously keep this attributedTo thing?
 -             "actor" => data["actor"] || data["attributedTo"],
 -             "object" => data
 -           },
 -           {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)},
 -           {:ok, activity} <- Transmogrifier.handle_incoming(params, options),
 -           {:object, _data, %Object{} = object} <-
 -             {:object, data, Object.normalize(activity, false)} do
 -        {:ok, object}
 -      else
 -        {:containment, _} ->
 -          {:error, "Object containment failed."}
 +      {:containment, _} ->
 +        {:error, "Object containment failed."}
  
 -        {:error, {:reject, nil}} ->
 -          {:reject, nil}
 +      {:error, {:reject, nil}} ->
 +        {:reject, nil}
  
 -        {:object, data, nil} ->
 -          reinject_object(%Object{}, data)
 +      {:object, data, nil} ->
-         reinject_object(data)
++        reinject_object(%Object{}, data)
  
 -        {:normalize, object = %Object{}} ->
 -          {:ok, object}
 +      {:normalize, object = %Object{}} ->
 +        {:ok, object}
  
 -        _e ->
 -          # Only fallback when receiving a fetch/normalization error with ActivityPub
 -          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 +      {:fetch_object, %Object{} = object} ->
 +        {:ok, object}
  
 -          # FIXME: OStatus Object Containment?
 -          case OStatus.fetch_activity_from_url(id) do
 -            {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)}
 -            e -> e
 -          end
 -      end
 +      _e ->
 +        # Only fallback when receiving a fetch/normalization error with ActivityPub
 +        Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 +
 +        # FIXME: OStatus Object Containment?
 +        case OStatus.fetch_activity_from_url(id) do
 +          {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)}
 +          e -> e
 +        end
      end
-     # end
    end
  
 +  defp prepare_activity_params(data) do
 +    %{
 +      "type" => "Create",
 +      "to" => data["to"],
 +      "cc" => data["cc"],
 +      # Should we seriously keep this attributedTo thing?
 +      "actor" => data["actor"] || data["attributedTo"],
 +      "object" => data
 +    }
 +  end
 +
    def fetch_object_from_id!(id, options \\ []) do
      with {:ok, object} <- fetch_object_from_id(id, options) do
        object
index 5b8e7f8f15d8568fad5ca587dc6735e37fd6ad2c,5878fb4f805625eda1a08febac7342471372de81..dad2fead8478c77ae4020e6639210d5486e58248
@@@ -15,6 -15,7 +15,7 @@@ defmodule Pleroma.Web.ActivityPub.Trans
    alias Pleroma.Web.ActivityPub.Utils
    alias Pleroma.Web.ActivityPub.Visibility
    alias Pleroma.Web.Federator
+   alias Pleroma.Workers.TransmogrifierWorker
  
    import Ecto.Query
  
@@@ -41,7 -42,8 +42,7 @@@
    end
  
    def fix_summary(%{"summary" => nil} = object) do
 -    object
 -    |> Map.put("summary", "")
 +    Map.put(object, "summary", "")
    end
  
    def fix_summary(%{"summary" => _} = object) do
      object
    end
  
 -  def fix_summary(object) do
 -    object
 -    |> Map.put("summary", "")
 -  end
 +  def fix_summary(object), do: Map.put(object, "summary", "")
  
    def fix_addressing_list(map, field) do
      cond do
          explicit_mentions,
          follower_collection
        ) do
 -    explicit_to =
 -      to
 -      |> Enum.filter(fn x -> x in explicit_mentions end)
 +    explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end)
  
 -    explicit_cc =
 -      to
 -      |> Enum.filter(fn x -> x not in explicit_mentions end)
 +    explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end)
  
      final_cc =
        (cc ++ explicit_cc)
    def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
  
    def fix_explicit_addressing(object) do
 -    explicit_mentions =
 -      object
 -      |> Utils.determine_explicit_mentions()
 +    explicit_mentions = Utils.determine_explicit_mentions(object)
  
 -    follower_collection = User.get_cached_by_ap_id(Containment.get_actor(object)).follower_address
 +    %User{follower_address: follower_collection} =
 +      object
 +      |> Containment.get_actor()
 +      |> User.get_cached_by_ap_id()
  
 -    explicit_mentions = explicit_mentions ++ [Pleroma.Constants.as_public(), follower_collection]
 +    explicit_mentions =
 +      explicit_mentions ++
 +        [
 +          Pleroma.Constants.as_public(),
 +          follower_collection
 +        ]
  
      fix_explicit_addressing(object, explicit_mentions, follower_collection)
    end
    end
  
    def fix_actor(%{"attributedTo" => actor} = object) do
 -    object
 -    |> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
 +    Map.put(object, "actor", Containment.get_actor(%{"actor" => actor}))
    end
  
    def fix_in_reply_to(object, options \\ [])
  
    def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
        when not is_nil(in_reply_to) do
 -    in_reply_to_id =
 -      cond do
 -        is_bitstring(in_reply_to) ->
 -          in_reply_to
 -
 -        is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) ->
 -          in_reply_to["id"]
 -
 -        is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) ->
 -          Enum.at(in_reply_to, 0)
 -
 -        # Maybe I should output an error too?
 -        true ->
 -          ""
 -      end
 -
 +    in_reply_to_id = prepare_in_reply_to(in_reply_to)
      object = Map.put(object, "inReplyToAtomUri", in_reply_to_id)
  
      if Federator.allowed_incoming_reply_depth?(options[:depth]) do
 -      case get_obj_helper(in_reply_to_id, options) do
 -        {:ok, replied_object} ->
 -          with %Activity{} = _activity <-
 -                 Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
 -            object
 -            |> Map.put("inReplyTo", replied_object.data["id"])
 -            |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
 -            |> Map.put("conversation", replied_object.data["context"] || object["conversation"])
 -            |> Map.put("context", replied_object.data["context"] || object["conversation"])
 -          else
 -            e ->
 -              Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
 -              object
 -          end
 -
 +      with {:ok, replied_object} <- get_obj_helper(in_reply_to_id, options),
 +           %Activity{} = _ <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
 +        object
 +        |> Map.put("inReplyTo", replied_object.data["id"])
 +        |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
 +        |> Map.put("conversation", replied_object.data["context"] || object["conversation"])
 +        |> Map.put("context", replied_object.data["context"] || object["conversation"])
 +      else
          e ->
            Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
            object
  
    def fix_in_reply_to(object, _options), do: object
  
 +  defp prepare_in_reply_to(in_reply_to) do
 +    cond do
 +      is_bitstring(in_reply_to) ->
 +        in_reply_to
 +
 +      is_map(in_reply_to) && is_bitstring(in_reply_to["id"]) ->
 +        in_reply_to["id"]
 +
 +      is_list(in_reply_to) && is_bitstring(Enum.at(in_reply_to, 0)) ->
 +        Enum.at(in_reply_to, 0)
 +
 +      true ->
 +        ""
 +    end
 +  end
 +
    def fix_context(object) do
      context = object["context"] || object["conversation"] || Utils.generate_context_id()
  
  
    def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do
      attachments =
 -      attachment
 -      |> Enum.map(fn data ->
 +      Enum.map(attachment, fn data ->
          media_type = data["mediaType"] || data["mimeType"]
          href = data["url"] || data["href"]
 -
          url = [%{"type" => "Link", "mediaType" => media_type, "href" => href}]
  
          data
          |> Map.put("url", url)
        end)
  
 -    object
 -    |> Map.put("attachment", attachments)
 +    Map.put(object, "attachment", attachments)
    end
  
    def fix_attachments(%{"attachment" => attachment} = object) when is_map(attachment) do
 -    Map.put(object, "attachment", [attachment])
 +    object
 +    |> Map.put("attachment", [attachment])
      |> fix_attachments()
    end
  
    def fix_attachments(object), do: object
  
    def fix_url(%{"url" => url} = object) when is_map(url) do
 -    object
 -    |> Map.put("url", url["href"])
 +    Map.put(object, "url", url["href"])
    end
  
    def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
      first_element = Enum.at(url, 0)
  
 -    link_element =
 -      url
 -      |> Enum.filter(fn x -> is_map(x) end)
 -      |> Enum.filter(fn x -> x["mimeType"] == "text/html" end)
 -      |> Enum.at(0)
 +    link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
  
      object
      |> Map.put("attachment", [first_element])
          true -> ""
        end
  
 -    object
 -    |> Map.put("url", url_string)
 +    Map.put(object, "url", url_string)
    end
  
    def fix_url(object), do: object
  
    def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do
 -    emoji = tags |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end)
 -
      emoji =
 -      emoji
 +      tags
 +      |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end)
        |> Enum.reduce(%{}, fn data, mapping ->
          name = String.trim(data["name"], ":")
  
 -        mapping |> Map.put(name, data["icon"]["url"])
 +        Map.put(mapping, name, data["icon"]["url"])
        end)
  
      # we merge mastodon and pleroma emoji into a single mapping, to allow for both wire formats
      emoji = Map.merge(object["emoji"] || %{}, emoji)
  
 -    object
 -    |> Map.put("emoji", emoji)
 +    Map.put(object, "emoji", emoji)
    end
  
    def fix_emoji(%{"tag" => %{"type" => "Emoji"} = tag} = object) do
      name = String.trim(tag["name"], ":")
      emoji = %{name => tag["icon"]["url"]}
  
 -    object
 -    |> Map.put("emoji", emoji)
 +    Map.put(object, "emoji", emoji)
    end
  
    def fix_emoji(object), do: object
        |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
        |> Enum.map(fn data -> String.slice(data["name"], 1..-1) end)
  
 -    combined = tag ++ tags
 -
 -    object
 -    |> Map.put("tag", combined)
 +    Map.put(object, "tag", tag ++ tags)
    end
  
    def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag} = tag} = object) do
      combined = [tag, String.slice(hashtag, 1..-1)]
  
 -    object
 -    |> Map.put("tag", combined)
 +    Map.put(object, "tag", combined)
    end
  
    def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag])
      content_groups = Map.to_list(content_map)
      {_, content} = Enum.at(content_groups, 0)
  
 -    object
 -    |> Map.put("content", content)
 +    Map.put(object, "content", content)
    end
  
    def fix_content_map(object), do: object
  
    def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options)
        when is_binary(reply_id) do
 -    reply =
 -      with true <- Federator.allowed_incoming_reply_depth?(options[:depth]),
 -           {:ok, object} <- get_obj_helper(reply_id, options) do
 -        object
 -      end
 -
 -    if reply && reply.data["type"] == "Question" do
 +    with true <- Federator.allowed_incoming_reply_depth?(options[:depth]),
 +         {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do
        Map.put(object, "type", "Answer")
      else
 -      object
 +      _ -> object
      end
    end
  
      end
    end
  
 +  # Reduce the object list to find the reported user.
 +  defp get_reported(objects) do
 +    Enum.reduce_while(objects, nil, fn ap_id, _ ->
 +      with %User{} = user <- User.get_cached_by_ap_id(ap_id) do
 +        {:halt, user}
 +      else
 +        _ -> {:cont, nil}
 +      end
 +    end)
 +  end
 +
    def handle_incoming(data, options \\ [])
  
    # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
      with context <- data["context"] || Utils.generate_context_id(),
           content <- data["content"] || "",
           %User{} = actor <- User.get_cached_by_ap_id(actor),
 -
           # Reduce the object list to find the reported user.
 -         %User{} = account <-
 -           Enum.reduce_while(objects, nil, fn ap_id, _ ->
 -             with %User{} = user <- User.get_cached_by_ap_id(ap_id) do
 -               {:halt, user}
 -             else
 -               _ -> {:cont, nil}
 -             end
 -           end),
 -
 +         %User{} = account <- get_reported(objects),
           # Remove the reported user from the object list.
           statuses <- Enum.filter(objects, fn ap_id -> ap_id != account.ap_id end) do
 -      params = %{
 +      %{
          actor: actor,
          context: context,
          account: account,
          statuses: statuses,
          content: content,
 -        additional: %{
 -          "cc" => [account.ap_id]
 -        }
 +        additional: %{"cc" => [account.ap_id]}
        }
 -
 -      ActivityPub.flag(params)
 +      |> ActivityPub.flag()
      end
    end
  
  
    def handle_incoming(_, _), do: :error
  
 +  @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil
    def get_obj_helper(id, options \\ []) do
 -    if object = Object.normalize(id, true, options), do: {:ok, object}, else: nil
 +    case Object.normalize(id, true, options) do
 +      %Object{} = object -> {:ok, object}
 +      _ -> nil
 +    end
    end
  
    def set_reply_to_uri(%{"inReplyTo" => in_reply_to} = object) when is_binary(in_reply_to) do
      {:ok, data}
    end
  
 -  def maybe_fix_object_url(data) do
 -    if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do
 -      case get_obj_helper(data["object"]) do
 -        {:ok, relative_object} ->
 -          if relative_object.data["external_url"] do
 -            _data =
 -              data
 -              |> Map.put("object", relative_object.data["external_url"])
 -          else
 -            data
 -          end
 -
 -        e ->
 -          Logger.error("Couldn't fetch #{data["object"]} #{inspect(e)}")
 -          data
 -      end
 +  def maybe_fix_object_url(%{"object" => object} = data) when is_binary(object) do
 +    with false <- String.starts_with?(object, "http"),
 +         {:fetch, {:ok, relative_object}} <- {:fetch, get_obj_helper(object)},
 +         %{data: %{"external_url" => external_url}} when not is_nil(external_url) <-
 +           relative_object do
 +      Map.put(data, "object", external_url)
      else
 -      data
 +      {:fetch, e} ->
 +        Logger.error("Couldn't fetch #{object} #{inspect(e)}")
 +        data
 +
 +      _ ->
 +        data
      end
    end
  
 +  def maybe_fix_object_url(data), do: data
 +
    def add_hashtags(object) do
      tags =
        (object["tag"] || [])
            tag
        end)
  
 -    object
 -    |> Map.put("tag", tags)
 +    Map.put(object, "tag", tags)
    end
  
    def add_mention_tags(object) do
      mentions =
        object
        |> Utils.get_notified_from_object()
 -      |> Enum.map(fn user ->
 -        %{"type" => "Mention", "href" => user.ap_id, "name" => "@#{user.nickname}"}
 -      end)
 +      |> Enum.map(&build_mention_tag/1)
  
      tags = object["tag"] || []
  
 -    object
 -    |> Map.put("tag", tags ++ mentions)
 +    Map.put(object, "tag", tags ++ mentions)
    end
  
 -  def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do
 -    user_info = add_emoji_tags(user_info)
 +  defp build_mention_tag(%{ap_id: ap_id, nickname: nickname} = _) do
 +    %{"type" => "Mention", "href" => ap_id, "name" => "@#{nickname}"}
 +  end
  
 -    object
 -    |> Map.put(:info, user_info)
 +  def take_emoji_tags(%User{info: %{emoji: emoji} = _user_info} = _user) do
 +    emoji
 +    |> Enum.flat_map(&Map.to_list/1)
 +    |> Enum.map(&build_emoji_tag/1)
    end
  
    # TODO: we should probably send mtime instead of unix epoch time for updated
    def add_emoji_tags(%{"emoji" => emoji} = object) do
      tags = object["tag"] || []
  
 -    out =
 -      emoji
 -      |> Enum.map(fn {name, url} ->
 -        %{
 -          "icon" => %{"url" => url, "type" => "Image"},
 -          "name" => ":" <> name <> ":",
 -          "type" => "Emoji",
 -          "updated" => "1970-01-01T00:00:00Z",
 -          "id" => url
 -        }
 -      end)
 +    out = Enum.map(emoji, &build_emoji_tag/1)
  
 -    object
 -    |> Map.put("tag", tags ++ out)
 +    Map.put(object, "tag", tags ++ out)
    end
  
 -  def add_emoji_tags(object) do
 -    object
 +  def add_emoji_tags(object), do: object
 +
 +  defp build_emoji_tag({name, url}) do
 +    %{
 +      "icon" => %{"url" => url, "type" => "Image"},
 +      "name" => ":" <> name <> ":",
 +      "type" => "Emoji",
 +      "updated" => "1970-01-01T00:00:00Z",
 +      "id" => url
 +    }
    end
  
    def set_conversation(object) do
  
    def add_attributed_to(object) do
      attributed_to = object["attributedTo"] || object["actor"]
 -
 -    object
 -    |> Map.put("attributedTo", attributed_to)
 +    Map.put(object, "attributedTo", attributed_to)
    end
  
    def prepare_attachments(object) do
          %{"url" => href, "mediaType" => media_type, "name" => data["name"], "type" => "Document"}
        end)
  
 -    object
 -    |> Map.put("attachment", attachments)
 +    Map.put(object, "attachment", attachments)
    end
  
    defp strip_internal_fields(object) do
      object
-     |> Map.drop([
-       "likes",
-       "like_count",
-       "announcements",
-       "announcement_count",
-       "emoji",
-       "context_id",
-       "deleted_activity_id"
-     ])
+     |> Map.drop(Pleroma.Constants.object_internal_fields())
    end
  
    defp strip_internal_tags(%{"tag" => tags} = object) do
 -    tags =
 -      tags
 -      |> Enum.filter(fn x -> is_map(x) end)
 +    tags = Enum.filter(tags, fn x -> is_map(x) end)
  
 -    object
 -    |> Map.put("tag", tags)
 +    Map.put(object, "tag", tags)
    end
  
    defp strip_internal_tags(object), do: object
      with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
           {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
           already_ap <- User.ap_enabled?(user),
-          {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do
-       unless already_ap do
-         PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user])
+          {:ok, user} <- upgrade_user(user, data) do
+       if not already_ap do
+         TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
        end
  
        {:ok, user}
      end
    end
  
+   defp upgrade_user(user, data) do
+     user
+     |> User.upgrade_changeset(data, true)
+     |> User.update_and_set_cache()
+   end
    def maybe_retire_websub(ap_id) do
      # some sanity checks
      if is_binary(ap_id) && String.length(ap_id) > 8 do
      end
    end
  
 -  def maybe_fix_user_url(data) do
 -    if is_map(data["url"]) do
 -      Map.put(data, "url", data["url"]["href"])
 -    else
 -      data
 -    end
 +  def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do
 +    Map.put(data, "url", url["href"])
    end
  
 -  def maybe_fix_user_object(data) do
 -    data
 -    |> maybe_fix_user_url
 -  end
 +  def maybe_fix_user_url(data), do: data
 +
 +  def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
  end
index 8abfa1fcd7543f77a901825eb491ad646178a8dd,164b973d0816dbc4fdc4b3a7a6a1f284c987bfd1..276306ec0f2bc474284826693fbf805c0397a9b2
@@@ -75,7 -75,10 +75,7 @@@ defmodule Pleroma.Web.ActivityPub.UserV
  
      endpoints = render("endpoints.json", %{user: user})
  
 -    user_tags =
 -      user
 -      |> Transmogrifier.add_emoji_tags()
 -      |> Map.get("tag", [])
 +    emoji_tags = Transmogrifier.take_emoji_tags(user)
  
      fields =
        user.info
        },
        "endpoints" => endpoints,
        "attachment" => fields,
 -      "tag" => (user.info.source_data["tag"] || []) ++ user_tags
 +      "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags
      }
      |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
      |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))
    end
  
    def render("following.json", %{user: user, page: page} = opts) do
-     showing = (opts[:for] && opts[:for] == user) || !user.info.hide_follows
+     showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_follows
+     showing_count = showing_items || !user.info.hide_follows_count
      query = User.get_friends_query(user)
      query = from(user in query, select: [:ap_id])
      following = Repo.all(query)
  
      total =
-       if showing do
+       if showing_count do
          length(following)
        else
          0
        end
  
-     collection(following, "#{user.ap_id}/following", page, showing, total)
+     collection(following, "#{user.ap_id}/following", page, showing_items, total)
      |> Map.merge(Utils.make_json_ld_header())
    end
  
    def render("following.json", %{user: user} = opts) do
-     showing = (opts[:for] && opts[:for] == user) || !user.info.hide_follows
+     showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_follows
+     showing_count = showing_items || !user.info.hide_follows_count
      query = User.get_friends_query(user)
      query = from(user in query, select: [:ap_id])
      following = Repo.all(query)
  
      total =
-       if showing do
+       if showing_count do
          length(following)
        else
          0
        "type" => "OrderedCollection",
        "totalItems" => total,
        "first" =>
-         if showing do
+         if showing_items do
            collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows)
          else
            "#{user.ap_id}/following?page=1"
    end
  
    def render("followers.json", %{user: user, page: page} = opts) do
-     showing = (opts[:for] && opts[:for] == user) || !user.info.hide_followers
+     showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_followers
+     showing_count = showing_items || !user.info.hide_followers_count
  
      query = User.get_followers_query(user)
      query = from(user in query, select: [:ap_id])
      followers = Repo.all(query)
  
      total =
-       if showing do
+       if showing_count do
          length(followers)
        else
          0
        end
  
-     collection(followers, "#{user.ap_id}/followers", page, showing, total)
+     collection(followers, "#{user.ap_id}/followers", page, showing_items, total)
      |> Map.merge(Utils.make_json_ld_header())
    end
  
    def render("followers.json", %{user: user} = opts) do
-     showing = (opts[:for] && opts[:for] == user) || !user.info.hide_followers
+     showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_followers
+     showing_count = showing_items || !user.info.hide_followers_count
  
      query = User.get_followers_query(user)
      query = from(user in query, select: [:ap_id])
      followers = Repo.all(query)
  
      total =
-       if showing do
+       if showing_count do
          length(followers)
        else
          0
        "type" => "OrderedCollection",
        "totalItems" => total,
        "first" =>
-         if showing do
-           collection(followers, "#{user.ap_id}/followers", 1, showing, total)
+         if showing_items do
+           collection(followers, "#{user.ap_id}/followers", 1, showing_items, total)
          else
            "#{user.ap_id}/followers?page=1"
          end
index 67f97247220975cc1f06c89ab00fbcb27c4a87a4,ebed65b7c950059c3ca2b008f7d70dfc0a7d77ac..a35db71dcff85cc594b4522d5b3e15fb88358593
@@@ -1,5 -1,5 +1,5 @@@
  # Pleroma: A lightweight social networking server
- # Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+ # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
  # SPDX-License-Identifier: AGPL-3.0-only
  
  defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
@@@ -8,6 -8,7 +8,7 @@@
    alias Pleroma.Object
    alias Pleroma.Object.Fetcher
    alias Pleroma.Repo
+   alias Pleroma.Tests.ObanHelpers
    alias Pleroma.User
    alias Pleroma.Web.ActivityPub.ActivityPub
    alias Pleroma.Web.ActivityPub.Transmogrifier
          |> Poison.decode!()
  
        {:ok, _} = Transmogrifier.handle_incoming(data)
+       ObanHelpers.perform_all()
  
        refute User.get_cached_by_ap_id(ap_id)
      end
        assert user.info.note_count == 1
  
        {:ok, user} = Transmogrifier.upgrade_user_from_ap_id("https://niu.moe/users/rye")
+       ObanHelpers.perform_all()
        assert user.info.ap_enabled
        assert user.info.note_count == 1
        assert user.follower_address == "https://niu.moe/users/rye/followers"
        refute recipient.follower_address in fixed_object["to"]
      end
    end
 +
 +  describe "fix_summary/1" do
 +    test "returns fixed object" do
 +      assert Transmogrifier.fix_summary(%{"summary" => nil}) == %{"summary" => ""}
 +      assert Transmogrifier.fix_summary(%{"summary" => "ok"}) == %{"summary" => "ok"}
 +      assert Transmogrifier.fix_summary(%{}) == %{"summary" => ""}
 +    end
 +  end
 +
 +  describe "fix_in_reply_to/2" do
 +    clear_config([:instance, :federation_incoming_replies_max_depth])
 +
 +    setup do
 +      data = Poison.decode!(File.read!("test/fixtures/mastodon-post-activity.json"))
 +      [data: data]
 +    end
 +
 +    test "returns not modified object when hasn't containts inReplyTo field", %{data: data} do
 +      assert Transmogrifier.fix_in_reply_to(data) == data
 +    end
 +
 +    test "returns object with inReplyToAtomUri when denied incoming reply", %{data: data} do
 +      Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0)
 +
 +      object_with_reply =
 +        Map.put(data["object"], "inReplyTo", "https://shitposter.club/notice/2827873")
 +
 +      modified_object = Transmogrifier.fix_in_reply_to(object_with_reply)
 +      assert modified_object["inReplyTo"] == "https://shitposter.club/notice/2827873"
 +      assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
 +
 +      object_with_reply =
 +        Map.put(data["object"], "inReplyTo", %{"id" => "https://shitposter.club/notice/2827873"})
 +
 +      modified_object = Transmogrifier.fix_in_reply_to(object_with_reply)
 +      assert modified_object["inReplyTo"] == %{"id" => "https://shitposter.club/notice/2827873"}
 +      assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
 +
 +      object_with_reply =
 +        Map.put(data["object"], "inReplyTo", ["https://shitposter.club/notice/2827873"])
 +
 +      modified_object = Transmogrifier.fix_in_reply_to(object_with_reply)
 +      assert modified_object["inReplyTo"] == ["https://shitposter.club/notice/2827873"]
 +      assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
 +
 +      object_with_reply = Map.put(data["object"], "inReplyTo", [])
 +      modified_object = Transmogrifier.fix_in_reply_to(object_with_reply)
 +      assert modified_object["inReplyTo"] == []
 +      assert modified_object["inReplyToAtomUri"] == ""
 +    end
 +
 +    test "returns modified object when allowed incoming reply", %{data: data} do
 +      object_with_reply =
 +        Map.put(
 +          data["object"],
 +          "inReplyTo",
 +          "https://shitposter.club/notice/2827873"
 +        )
 +
 +      Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 5)
 +      modified_object = Transmogrifier.fix_in_reply_to(object_with_reply)
 +
 +      assert modified_object["inReplyTo"] ==
 +               "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
 +
 +      assert modified_object["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
 +
 +      assert modified_object["conversation"] ==
 +               "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26"
 +
 +      assert modified_object["context"] ==
 +               "tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26"
 +    end
 +  end
 +
 +  describe "fix_url/1" do
 +    test "fixes data for object when url is map" do
 +      object = %{
 +        "url" => %{
 +          "type" => "Link",
 +          "mimeType" => "video/mp4",
 +          "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4"
 +        }
 +      }
 +
 +      assert Transmogrifier.fix_url(object) == %{
 +               "url" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4"
 +             }
 +    end
 +
 +    test "fixes data for video object" do
 +      object = %{
 +        "type" => "Video",
 +        "url" => [
 +          %{
 +            "type" => "Link",
 +            "mimeType" => "video/mp4",
 +            "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4"
 +          },
 +          %{
 +            "type" => "Link",
 +            "mimeType" => "video/mp4",
 +            "href" => "https://peertube46fb-ad81-2d4c2d1630e3-240.mp4"
 +          },
 +          %{
 +            "type" => "Link",
 +            "mimeType" => "text/html",
 +            "href" => "https://peertube.-2d4c2d1630e3"
 +          },
 +          %{
 +            "type" => "Link",
 +            "mimeType" => "text/html",
 +            "href" => "https://peertube.-2d4c2d16377-42"
 +          }
 +        ]
 +      }
 +
 +      assert Transmogrifier.fix_url(object) == %{
 +               "attachment" => [
 +                 %{
 +                   "href" => "https://peede8d-46fb-ad81-2d4c2d1630e3-480.mp4",
 +                   "mimeType" => "video/mp4",
 +                   "type" => "Link"
 +                 }
 +               ],
 +               "type" => "Video",
 +               "url" => "https://peertube.-2d4c2d1630e3"
 +             }
 +    end
 +
 +    test "fixes url for not Video object" do
 +      object = %{
 +        "type" => "Text",
 +        "url" => [
 +          %{
 +            "type" => "Link",
 +            "mimeType" => "text/html",
 +            "href" => "https://peertube.-2d4c2d1630e3"
 +          },
 +          %{
 +            "type" => "Link",
 +            "mimeType" => "text/html",
 +            "href" => "https://peertube.-2d4c2d16377-42"
 +          }
 +        ]
 +      }
 +
 +      assert Transmogrifier.fix_url(object) == %{
 +               "type" => "Text",
 +               "url" => "https://peertube.-2d4c2d1630e3"
 +             }
 +
 +      assert Transmogrifier.fix_url(%{"type" => "Text", "url" => []}) == %{
 +               "type" => "Text",
 +               "url" => ""
 +             }
 +    end
 +
 +    test "retunrs not modified object" do
 +      assert Transmogrifier.fix_url(%{"type" => "Text"}) == %{"type" => "Text"}
 +    end
 +  end
 +
 +  describe "get_obj_helper/2" do
 +    test "returns nil when cannot normalize object" do
 +      refute Transmogrifier.get_obj_helper("test-obj-id")
 +    end
 +
 +    test "returns {:ok, %Object{}} for success case" do
 +      assert {:ok, %Object{}} =
 +               Transmogrifier.get_obj_helper("https://shitposter.club/notice/2827873")
 +    end
 +  end
 +
 +  describe "fix_attachments/1" do
 +    test "returns not modified object" do
 +      data = Poison.decode!(File.read!("test/fixtures/mastodon-post-activity.json"))
 +      assert Transmogrifier.fix_attachments(data) == data
 +    end
 +
 +    test "returns modified object when attachment is map" do
 +      assert Transmogrifier.fix_attachments(%{
 +               "attachment" => %{
 +                 "mediaType" => "video/mp4",
 +                 "url" => "https://peertube.moe/stat-480.mp4"
 +               }
 +             }) == %{
 +               "attachment" => [
 +                 %{
 +                   "mediaType" => "video/mp4",
 +                   "url" => [
 +                     %{
 +                       "href" => "https://peertube.moe/stat-480.mp4",
 +                       "mediaType" => "video/mp4",
 +                       "type" => "Link"
 +                     }
 +                   ]
 +                 }
 +               ]
 +             }
 +    end
 +
 +    test "returns modified object when attachment is list" do
 +      assert Transmogrifier.fix_attachments(%{
 +               "attachment" => [
 +                 %{"mediaType" => "video/mp4", "url" => "https://pe.er/stat-480.mp4"},
 +                 %{"mimeType" => "video/mp4", "href" => "https://pe.er/stat-480.mp4"}
 +               ]
 +             }) == %{
 +               "attachment" => [
 +                 %{
 +                   "mediaType" => "video/mp4",
 +                   "url" => [
 +                     %{
 +                       "href" => "https://pe.er/stat-480.mp4",
 +                       "mediaType" => "video/mp4",
 +                       "type" => "Link"
 +                     }
 +                   ]
 +                 },
 +                 %{
 +                   "href" => "https://pe.er/stat-480.mp4",
 +                   "mediaType" => "video/mp4",
 +                   "mimeType" => "video/mp4",
 +                   "url" => [
 +                     %{
 +                       "href" => "https://pe.er/stat-480.mp4",
 +                       "mediaType" => "video/mp4",
 +                       "type" => "Link"
 +                     }
 +                   ]
 +                 }
 +               ]
 +             }
 +    end
 +  end
 +
 +  describe "fix_emoji/1" do
 +    test "returns not modified object when object not contains tags" do
 +      data = Poison.decode!(File.read!("test/fixtures/mastodon-post-activity.json"))
 +      assert Transmogrifier.fix_emoji(data) == data
 +    end
 +
 +    test "returns object with emoji when object contains list tags" do
 +      assert Transmogrifier.fix_emoji(%{
 +               "tag" => [
 +                 %{"type" => "Emoji", "name" => ":bib:", "icon" => %{"url" => "/test"}},
 +                 %{"type" => "Hashtag"}
 +               ]
 +             }) == %{
 +               "emoji" => %{"bib" => "/test"},
 +               "tag" => [
 +                 %{"icon" => %{"url" => "/test"}, "name" => ":bib:", "type" => "Emoji"},
 +                 %{"type" => "Hashtag"}
 +               ]
 +             }
 +    end
 +
 +    test "returns object with emoji when object contains map tag" do
 +      assert Transmogrifier.fix_emoji(%{
 +               "tag" => %{"type" => "Emoji", "name" => ":bib:", "icon" => %{"url" => "/test"}}
 +             }) == %{
 +               "emoji" => %{"bib" => "/test"},
 +               "tag" => %{"icon" => %{"url" => "/test"}, "name" => ":bib:", "type" => "Emoji"}
 +             }
 +    end
 +  end
  end
index 4390f9272bc316a520bf481415ff4ed649235d9c,2b4a04afdf516d16d0edb9e0600123ad8f51f6f8..8878b8ea4e5017ff9e848621ec8fdaf67c085cc2
@@@ -37,22 -37,6 +37,22 @@@ defmodule Pleroma.Web.ActivityPub.UserV
             } = UserView.render("user.json", %{user: user})
    end
  
 +  test "Renders with emoji tags" do
 +    user = insert(:user, %{info: %{emoji: [%{"bib" => "/test"}]}})
 +
 +    assert %{
 +             "tag" => [
 +               %{
 +                 "icon" => %{"type" => "Image", "url" => "/test"},
 +                 "id" => "/test",
 +                 "name" => ":bib:",
 +                 "type" => "Emoji",
 +                 "updated" => "1970-01-01T00:00:00Z"
 +               }
 +             ]
 +           } = UserView.render("user.json", %{user: user})
 +  end
 +
    test "Does not add an avatar image if the user hasn't set one" do
      user = insert(:user)
      {:ok, user} = User.ensure_keys_present(user)
        other_user = insert(:user)
        {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
        assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user})
-       info = Map.put(user.info, :hide_followers, true)
+       info = Map.merge(user.info, %{hide_followers_count: true, hide_followers: true})
        user = Map.put(user, :info, info)
        assert %{"totalItems" => 0} = UserView.render("followers.json", %{user: user})
      end
+     test "sets correct totalItems when followers are hidden but the follower counter is not" do
+       user = insert(:user)
+       other_user = insert(:user)
+       {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user)
+       assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user})
+       info = Map.merge(user.info, %{hide_followers_count: false, hide_followers: true})
+       user = Map.put(user, :info, info)
+       assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user})
+     end
    end
  
    describe "following" do
        other_user = insert(:user)
        {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user)
        assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user})
-       info = Map.put(user.info, :hide_follows, true)
+       info = Map.merge(user.info, %{hide_follows_count: true, hide_follows: true})
        user = Map.put(user, :info, info)
        assert %{"totalItems" => 0} = UserView.render("following.json", %{user: user})
      end
+     test "sets correct totalItems when follows are hidden but the follow counter is not" do
+       user = insert(:user)
+       other_user = insert(:user)
+       {:ok, user, _other_user, _activity} = CommonAPI.follow(user, other_user)
+       assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user})
+       info = Map.merge(user.info, %{hide_follows_count: false, hide_follows: true})
+       user = Map.put(user, :info, info)
+       assert %{"totalItems" => 1} = UserView.render("following.json", %{user: user})
+     end
    end
  end