Make Floki use fast_html

diff --git a/config/config.exs b/config/config.exs
index 41c1ff6371a21e56af6a4fdaf45aaf79801096c1..364aaf7769085532ad9046d6383a251a98d36519 100644 (file)
--- a/config/config.exs
+++ b/config/config.exs
@@ -612,6 +612,8 @@ config :pleroma, :modules, runtime_dir: "instance/modules"
 
 config :pleroma, configurable_from_database: false
 
+config :floki, :html_parser, Floki.HTMLParser.FastHtml
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 11513106eb643769c72701865459cd9eb70a22dc..05946aa966a829deea50a8b33d54310f287c484e 100644 (file)
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -108,6 +108,7 @@ defmodule Pleroma.HTML do
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       result =
         content
+        |> Floki.parse_fragment!()
         |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]")
         |> Floki.attribute("a", "href")
         |> Enum.at(0)

diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
index 8abe18e29530db74d58b4d9766c209d4026b4109..802d10edcdc63daf23d9a3a79719cae3db1ad706 100644 (file)
--- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
@@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
   # does the post contain links?
   defp contains_links?(%{"content" => content} = _object) do
     content
+    |> Floki.parse_fragment!()
     |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"],a.zrl")
     |> Floki.attribute("a", "href")
     |> length() > 0

diff --git a/lib/pleroma/web/metadata/rel_me.ex b/lib/pleroma/web/metadata/rel_me.ex
index f87fc197364f2588d6510ca0237ae1ffe5c7570c..86dcc1a3b6c7c102ed767d543692e7dbc91819da 100644 (file)
--- a/lib/pleroma/web/metadata/rel_me.ex
+++ b/lib/pleroma/web/metadata/rel_me.ex
@@ -8,8 +8,10 @@ defmodule Pleroma.Web.Metadata.Providers.RelMe do
 
   @impl Provider
   def build_tags(%{user: user}) do
-    (Floki.attribute(user.bio, "link[rel~=me]", "href") ++
-       Floki.attribute(user.bio, "a[rel~=me]", "href"))
+    bio_tree = Floki.parse_fragment!(user.bio)
+
+    (Floki.attribute(bio_tree, "link[rel~=me]", "href") ++
+       Floki.attribute(bio_tree, "a[rel~=me]", "href"))
     |> Enum.map(fn link ->
       {:link, [rel: "me", href: link], []}
     end)

diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex
index 16b1a53d2886df63a1fa3ce2354b85b53156aa6b..540fa65dffba1ce8702509acac42b065f214390a 100644 (file)
--- a/lib/pleroma/web/rel_me.ex
+++ b/lib/pleroma/web/rel_me.ex
@@ -27,9 +27,10 @@ defmodule Pleroma.Web.RelMe do
   defp parse_url(url) do
     with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
            Pleroma.HTTP.get(url, [], adapter: @hackney_options),
+         {:ok, html_tree} <- Floki.parse_document(html),
          data <-
-           Floki.attribute(html, "link[rel~=me]", "href") ++
-             Floki.attribute(html, "a[rel~=me]", "href") do
+           Floki.attribute(html_tree, "link[rel~=me]", "href") ++
+             Floki.attribute(html_tree, "a[rel~=me]", "href") do
       {:ok, data}
     end
   rescue

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index c06b0a0f2668338415d592fde838bff42e49e056..9702e90f19b8ed02ce5569b2576731e204d0ab00 100644 (file)
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -81,18 +81,18 @@ defmodule Pleroma.Web.RichMedia.Parser do
       {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
 
       html
-      |> parse_html
+      |> parse_html()
       |> maybe_parse()
       |> Map.put(:url, url)
       |> clean_parsed_data()
       |> check_parsed_data()
     rescue
       e ->
-        {:error, "Parsing error: #{inspect(e)}"}
+        {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
     end
   end
 
-  defp parse_html(html), do: Floki.parse(html)
+  defp parse_html(html), do: Floki.parse_document!(html)
 
   defp maybe_parse(html) do
     Enum.reduce_while(parsers(), %{}, fn parser, acc ->

diff --git a/test/web/rich_media/parsers/twitter_card_test.exs b/test/web/rich_media/parsers/twitter_card_test.exs
index 751ca614c95639cf4d90ee75c8c7d6b15a8e1b64..f2ebbde7e6fc734cbe92b65b98f28063a4e1ad12 100644 (file)
--- a/test/web/rich_media/parsers/twitter_card_test.exs
+++ b/test/web/rich_media/parsers/twitter_card_test.exs
@@ -7,11 +7,14 @@ defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
   alias Pleroma.Web.RichMedia.Parsers.TwitterCard
 
   test "returns error when html not contains twitter card" do
-    assert TwitterCard.parse("", %{}) == {:error, "No twitter card metadata found"}
+    assert TwitterCard.parse([{"html", [], [{"head", [], []}, {"body", [], []}]}], %{}) ==
+             {:error, "No twitter card metadata found"}
   end
 
   test "parses twitter card with only name attributes" do
-    html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers3.html")
+    html =
+      File.read!("test/fixtures/nypd-facial-recognition-children-teenagers3.html")
+      |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
@@ -26,7 +29,9 @@ defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
   end
 
   test "parses twitter card with only property attributes" do
-    html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers2.html")
+    html =
+      File.read!("test/fixtures/nypd-facial-recognition-children-teenagers2.html")
+      |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
@@ -45,7 +50,9 @@ defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
   end
 
   test "parses twitter card with name & property attributes" do
-    html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html")
+    html =
+      File.read!("test/fixtures/nypd-facial-recognition-children-teenagers.html")
+      |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
@@ -73,7 +80,8 @@ defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
         "YTQ5MF9EQVIgZXhodW1hdGlvbiBvZiBNYXJnYXJldCBDb3JiaW4gZ3JhdmUgMTkyNi5qcGciXSxbInAiLCJjb252ZXJ0IiwiIl0sWyJwIiwiY29udmVydCIsIi1xdWFsaXR5IDgxIC1hdXRvLW9" <>
         "yaWVudCJdLFsicCIsInRodW1iIiwiNjAweD4iXV0/DAR%20exhumation%20of%20Margaret%20Corbin%20grave%201926.jpg"
 
-    html = File.read!("test/fixtures/margaret-corbin-grave-west-point.html")
+    html =
+      File.read!("test/fixtures/margaret-corbin-grave-west-point.html") |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
              {:ok,
@@ -87,7 +95,9 @@ defmodule Pleroma.Web.RichMedia.Parsers.TwitterCardTest do
   end
 
   test "takes first founded title in html head if there is html markup error" do
-    html = File.read!("test/fixtures/nypd-facial-recognition-children-teenagers4.html")
+    html =
+      File.read!("test/fixtures/nypd-facial-recognition-children-teenagers4.html")
+      |> Floki.parse_document!()
 
     assert TwitterCard.parse(html, %{}) ==
              {:ok,