### Changed
-- **Breaking:** Changed storage of hashtags in plain-text to `object->hashtags`, run [`pleroma.database fill_old_hashtags` mix task](docs/administration/CLI_tasks/database.md) for old objects (works while pleroma is running).
- Polls now always return a `voters_count`, even if they are single-choice.
- Admin Emails: The ap id is used as the user link in emails now.
- Improved registration workflow for email confirmation and account approval modes.
- Static-FE: Fix remote posts not being sanitized
### Fixed
+=======
- Rate limiter crashes when there is no explicitly specified ip in the config
- 500 errors when no `Accept` header is present if Static-FE is enabled
- Instance panel not being updated immediately due to wrong `Cache-Control` headers
mix pleroma.database fix_likes_collections
```
-## Fill hashtags for old objects
-
-Migrate hashags fields for old objects, from now to `months_limit` months.
-
-```sh tab="OTP"
-./bin/pleroma_ctl database fill_old_hashtags <months_limit>
-```
-
-```sh tab="From Source"
-mix pleroma.database fill_old_hashtags <months_limit>
-```
-
## Vacuum the database
### Analyze
|> Stream.run()
end
- def run(["fill_old_hashtags", month_limit]) do
- import Ecto.Query
-
- start_pleroma()
-
- month_limit = String.to_integer(month_limit)
-
- if month_limit < 1 do
- shell_error("Invalid `month_limit` argument, needs to be greater than 1")
- else
- time_limit = DateTime.utc_now() |> Timex.shift(months: -month_limit)
-
- from(
- o in Object,
- where: fragment("(?)->>'hashtags' is null", o.data),
- where: fragment("(?)->>'tag' != '[]'", o.data),
- where: o.inserted_at < ^time_limit,
- select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)}
- )
- |> Pleroma.Repo.chunk_stream(200, :batches)
- |> Stream.each(fn objects ->
- Repo.transaction(fn ->
- objects_first = objects |> List.first()
- objects_last = objects |> List.last()
-
- Logger.info(
- "fill_old_hashtags: #{objects_first.id} (#{objects_first.inserted_at}) -- #{
- objects_last.id
- } (#{objects_last.inserted_at})"
- )
-
- objects
- |> Enum.map(fn object ->
- tags =
- object.tag
- |> Jason.decode!()
- |> Enum.filter(&is_bitstring(&1))
-
- Object
- |> where([o], o.id == ^object.id)
- |> update([o],
- set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)]
- )
- |> Repo.update_all([], timeout: :infinity)
- end)
- end)
- end)
- |> Stream.run()
- end
- end
-
def run(["vacuum", args]) do
start_pleroma()
tags
end
- defp hashtags_to_topics(object) do
- object
- |> Object.hashtags()
+ defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
+ tags
+ |> Enum.filter(&is_bitstring(&1))
|> Enum.map(fn tag -> "hashtag:" <> tag end)
end
+ defp hashtags_to_topics(_), do: []
+
defp remote_topics(%{local: true}), do: []
defp remote_topics(%{actor: actor}) when is_binary(actor),
"emoji",
"context_id",
"deleted_activity_id",
- "pleroma_internal",
- "hashtags"
+ "pleroma_internal"
]
)
def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true))
-
- def hashtags(%Object{data: %{"hashtags" => hashtags}}), do: hashtags || []
- def hashtags(%Object{data: %{"tag" => tags}}), do: Enum.filter(tags, &is_bitstring(&1))
- def hashtags(_), do: []
end
defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
from(
[_activity, object] in query,
- where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject)
+ where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
)
end
defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
from(
[_activity, object] in query,
- where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all)
+ where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
)
end
defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
from(
[_activity, object] in query,
- where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag)
+ where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
)
end
defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
from(
[_activity, object] in query,
- where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag)
+ where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
)
end
alias Pleroma.Config
alias Pleroma.FollowingRelationship
- alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.MRF
object =
if MRF.subdomain_match?(media_nsfw, actor_host) do
- child_object =
- child_object
- |> Map.put("hashtags", Object.hashtags(%Object{data: child_object}) ++ ["nsfw"])
- |> Map.put("sensitive", true)
-
+ tags = (child_object["tag"] || []) ++ ["nsfw"]
+ child_object = Map.put(child_object, "tag", tags)
+ child_object = Map.put(child_object, "sensitive", true)
Map.put(object, "object", child_object)
else
object
def fix_emoji(object), do: object
def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
- hashtags =
+ tags =
tag
|> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
- |> Enum.map(fn
- %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
- %{"name" => hashtag} -> String.downcase(hashtag)
+ |> Enum.map(fn %{"name" => name} ->
+ name
+ |> String.slice(1..-1)
+ |> String.downcase()
end)
- Map.put(object, "hashtags", hashtags)
+ Map.put(object, "tag", tag ++ tags)
end
def fix_tag(%{"tag" => %{} = tag} = object) do
def maybe_fix_object_url(data), do: data
def add_hashtags(object) do
- hashtags =
- %Object{data: object}
- |> Object.hashtags()
- |> Enum.map(fn tag ->
- %{
- "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}",
- "name" => "##{tag}",
- "type" => "Hashtag"
- }
+ tags =
+ (object["tag"] || [])
+ |> Enum.map(fn
+ # Expand internal representation tags into AS2 tags.
+ tag when is_binary(tag) ->
+ %{
+ "href" => Pleroma.Web.Endpoint.url() <> "/tags/#{tag}",
+ "name" => "##{tag}",
+ "type" => "Hashtag"
+ }
+
+ # Do not process tags which are already AS2 tag objects.
+ tag when is_map(tag) ->
+ tag
end)
- Map.put(object, "tag", hashtags ++ (object["tag"] || []))
+ Map.put(object, "tag", tags)
end
# TODO These should be added on our side on insertion, it doesn't make much
end
def set_sensitive(object) do
- tags = object["hashtags"] || object["tag"] || []
+ tags = object["tag"] || []
Map.put(object, "sensitive", "nsfw" in tags)
end
"context" => draft.context,
"attachment" => draft.attachments,
"actor" => draft.user.ap_id,
- "tag" => Enum.filter(draft.tags, &is_map(&1)) |> Enum.uniq(),
- "hashtags" =>
- draft.tags
- |> Enum.reduce([], fn
- # Why so many formats
- {:name, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
- {"#" <> _, x}, acc -> if is_bitstring(x), do: [x | acc], else: acc
- x, acc -> if is_bitstring(x), do: [x | acc], else: acc
- end)
- |> Enum.uniq()
+ "tag" => Keyword.values(draft.tags) |> Enum.uniq()
}
|> add_in_reply_to(draft.in_reply_to)
|> Map.merge(draft.extra)
%{
activity: activity,
- object: object,
data: Map.get(object, :data),
actor: actor
}
media_attachments: attachments,
poll: render(PollView, "show.json", object: object, for: opts[:for]),
mentions: mentions,
- tags: build_tags(Object.hashtags(object)),
+ tags: build_tags(tags),
application: %{
name: "Web",
website: nil
<link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
<% end %>
- <%= for hashtag <- Object.hashtags(@object) do %>
- <category term="<%= hashtag %>"></category>
+ <%= for tag <- @data["tag"] || [] do %>
+ <category term="<%= tag %>"></category>
<% end %>
<%= for attachment <- @data["attachment"] || [] do %>
<link><%= @data["external_url"] %></link>
<% end %>
- <%= for hashtag <- Object.hashtags(@object) do %>
- <category term="<%= hashtag %>"></category>
+ <%= for tag <- @data["tag"] || [] do %>
+ <category term="<%= tag %>"></category>
<% end %>
<%= for attachment <- @data["attachment"] || [] do %>
<% end %>
<% end %>
- <%= for hashtag <- Object.hashtags(@object) do %>
- <category term="<%= hashtag %>"></category>
+ <%= for tag <- @data["tag"] || [] do %>
+ <category term="<%= tag %>"></category>
<% end %>
<%= for {emoji, file} <- @data["emoji"] || %{} do %>
+++ /dev/null
-defmodule Pleroma.Repo.Migrations.AddHashtagsIndexToObjects do
- use Ecto.Migration
-
- def change do
- drop_if_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags))
-
- create_if_not_exists(
- index(:objects, ["(data->'hashtags')"], using: :gin, name: :objects_hashtags)
- )
- end
-end
end
test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do
- tagged_data = Map.put(data, "hashtags", ["foo", "bar"])
+ tagged_data = Map.put(data, "tag", ["foo", "bar"])
activity = %{activity | object: %{object | data: tagged_data}}
topics = Topics.get_activity_topics(activity)
assert SimplePolicy.filter(media_message) ==
{:ok,
media_message
- |> put_in(["object", "hashtags"], ["foo", "nsfw"])
+ |> put_in(["object", "tag"], ["foo", "nsfw"])
|> put_in(["object", "sensitive"], true)}
assert SimplePolicy.filter(local_message) == {:ok, local_message}
assert SimplePolicy.filter(media_message) ==
{:ok,
media_message
- |> put_in(["object", "hashtags"], ["foo", "nsfw"])
+ |> put_in(["object", "tag"], ["foo", "nsfw"])
|> put_in(["object", "sensitive"], true)}
assert SimplePolicy.filter(local_message) == {:ok, local_message}
"type" => "Create",
"object" => %{
"attachment" => [%{}],
- "hashtags" => ["foo"],
+ "tag" => ["foo"],
"sensitive" => false
}
}
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
- assert ["test"] == object.data["hashtags"]
+ assert "test" in object.data["tag"]
end
test "it cleans up incoming notices which are not really DMs" do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"])
- assert object.data["hashtags"] == ["moo"]
+ assert Enum.at(object.data["tag"], 2) == "moo"
end
test "it works for incoming notices with contentMap" do
{:ok, activity} = CommonAPI.post(user, %{status: "#2hu :firefox:"})
- {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data)
-
- assert [
- %{"name" => "#2hu", "type" => "Hashtag"},
- %{"name" => ":firefox:", "type" => "Emoji"}
- ] = modified_object["tag"]
-
- refute Map.has_key?(modified_object, "hashtags")
- refute Map.has_key?(modified_object, "emoji")
- refute Map.has_key?(modified_object, "like_count")
- refute Map.has_key?(modified_object, "announcements")
- refute Map.has_key?(modified_object, "announcement_count")
- refute Map.has_key?(modified_object, "context_id")
+ {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
+
+ assert length(modified["object"]["tag"]) == 2
+
+ assert is_nil(modified["object"]["emoji"])
+ assert is_nil(modified["object"]["like_count"])
+ assert is_nil(modified["object"]["announcements"])
+ assert is_nil(modified["object"]["announcement_count"])
+ assert is_nil(modified["object"]["context_id"])
end
test "it strips internal fields of article" do
activity = insert(:article_activity)
- {:ok, %{"object" => modified_object}} = Transmogrifier.prepare_outgoing(activity.data)
+ {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
- assert [
- %{"name" => "#2hu", "type" => "Hashtag"},
- %{"name" => ":2hu:", "type" => "Emoji"}
- ] = modified_object["tag"]
+ assert length(modified["object"]["tag"]) == 2
- refute Map.has_key?(modified_object, "hashtags")
- refute Map.has_key?(modified_object, "emoji")
- refute Map.has_key?(modified_object, "like_count")
- refute Map.has_key?(modified_object, "announcements")
- refute Map.has_key?(modified_object, "announcement_count")
- refute Map.has_key?(modified_object, "context_id")
+ assert is_nil(modified["object"]["emoji"])
+ assert is_nil(modified["object"]["like_count"])
+ assert is_nil(modified["object"]["announcements"])
+ assert is_nil(modified["object"]["announcement_count"])
+ assert is_nil(modified["object"]["context_id"])
+ assert is_nil(modified["object"]["likes"])
end
test "the directMessage flag is present" do
"context" => "2hu",
"sensitive" => false,
"summary" => "test summary",
- "hashtags" => ["jimm"],
- "tag" => [],
+ "tag" => ["jimm"],
"to" => [user2.ap_id],
"type" => "Note",
"custom_tag" => "test"
object = Object.normalize(activity)
- assert object.data["tag"] == []
- assert object.data["hashtags"] == ["2hu"]
+ assert object.data["tag"] == ["2hu"]
end
test "it adds emoji in the object" do
mentions: [],
tags: [
%{
- name: "2hu",
- url: "/tag/2hu"
+ name: "#{object_data["tag"]}",
+ url: "/tag/#{object_data["tag"]}"
}
],
application: %{
"like_count" => 0,
"context" => "2hu",
"summary" => "2hu",
- "hashtags" => ["2hu"],
+ "tag" => ["2hu"],
"emoji" => %{
"2hu" => "corndog.png"
}
projects / akkoma / commitdiff