Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into websearch

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9a754ed782563e32b897af74935f9b6c0c5dd770..1b05e4a08ec58e7377f25705b0e0067f7dc8c7af 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -57,7 +57,7 @@ unit-testing:
     policy: pull
 
   services:
-  - name: postgres:9.6
+  - name: postgres:13
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 281f067294eaed80d40ae58bb50299da54d6ffcd..f4ef66408574649af2c092a8d0662912f27afc46 100644 (file)
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Polls now always return a `voters_count`, even if they are single-choice.
 - Admin Emails: The ap id is used as the user link in emails now.
+- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
 
 ### Added
 

diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
index 382c81118c964b5ee8448477426d328ffa9f2764..ea9783225129852f05f849049843ac7d0cea5f56 100644 (file)
--- a/lib/pleroma/activity/search.ex
+++ b/lib/pleroma/activity/search.ex
@@ -19,11 +19,18 @@ defmodule Pleroma.Activity.Search do
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
+    search_function =
+      if Application.get_env(:postgres, :version) >= 11 do
+        :websearch
+      else
+        :plain
+      end
+
     Activity
     |> Activity.with_preloaded_object()
     |> Activity.restrict_deactivated_users()
     |> restrict_public()
-    |> query_with(index_type, search_query)
+    |> query_with(index_type, search_query, search_function)
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
     |> maybe_restrict_blocked(user)
@@ -53,7 +60,7 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
-  defp query_with(q, :gin, search_query) do
+  defp query_with(q, :gin, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -64,7 +71,18 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
-  defp query_with(q, :rum, search_query) do
+  defp query_with(q, :gin, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
+          o.data,
+          ^search_query
+        )
+    )
+  end
+
+  defp query_with(q, :rum, search_query, :plain) do
     from([a, o] in q,
       where:
         fragment(
@@ -76,6 +94,18 @@ defmodule Pleroma.Activity.Search do
     )
   end
 
+  defp query_with(q, :rum, search_query, :websearch) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ websearch_to_tsquery('english', ?)",
+          o.fts_content,
+          ^search_query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   defp maybe_restrict_local(q, user) do
     limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
 

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index ced14f87ff510b68b9531f263b400b16cdd234c5..22936bd7f245e7f7c128b1c7581c0f87bd4d4d82 100644 (file)
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -110,7 +110,28 @@ defmodule Pleroma.Application do
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
-    Supervisor.start_link(children, opts)
+    result = Supervisor.start_link(children, opts)
+
+    set_postgres_server_version()
+
+    result
+  end
+
+  defp set_postgres_server_version do
+    version =
+      with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
+           {num, _} <- Float.parse(version) do
+        num
+      else
+        e ->
+          Logger.warn(
+            "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
+          )
+
+          9.6
+      end
+
+    Application.put_env(:postgres, :version, version)
   end
 
   def load_custom_modules do

diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/activity/search_test.exs
new file mode 100644 (file)
index 0000000..37c0fee
--- /dev/null
+++ b/test/pleroma/activity/search_test.exs
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.SearchTest do
+  alias Pleroma.Activity.Search
+  alias Pleroma.Web.CommonAPI
+  import Pleroma.Factory
+
+  use Pleroma.DataCase
+
+  test "it finds something" do
+    user = insert(:user)
+    {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+
+    [result] = Search.search(nil, "wednesday")
+
+    assert result.id == post.id
+  end
+
+  test "using plainto_tsquery on postgres < 11" do
+    old_config = Application.get_env(:postgres, :version)
+    Application.put_env(:postgres, :version, 10.0)
+
+    user = insert(:user)
+    {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+    {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+    # plainto doesn't understand complex queries
+    assert [result] = Search.search(nil, "wednesday -dudes")
+
+    assert result.id == post.id
+
+    Application.put_env(:postgres, :version, old_config)
+  end
+
+  test "using websearch_to_tsquery" do
+    user = insert(:user)
+    {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+    {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+    assert [result] = Search.search(nil, "wednesday -dudes")
+
+    assert result.id == other_post.id
+  end
+end

diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
index 04dc6f4458cf86156090b9c95f25d47e50ffc850..2b2579857a2e5db6d8aa76280d356481d62e5578 100644 (file)
--- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
@@ -279,6 +279,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
     end
 
     test "search fetches remote statuses and prefers them over other results", %{conn: conn} do
+      old_config = Application.get_env(:postgres, :version)
+      Application.put_env(:postgres, :version, 10.0)
+
       capture_log(fn ->
         {:ok, %{id: activity_id}} =
           CommonAPI.post(insert(:user), %{
@@ -295,6 +298,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
                  %{"id" => ^activity_id}
                ] = results["statuses"]
       end)
+
+      Application.put_env(:postgres, :version, old_config)
     end
 
     test "search doesn't show statuses that it shouldn't", %{conn: conn} do