From c3fde9577d5822243a7567a55cefa2fd2f289559 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Tue, 30 Aug 2022 14:58:54 +0000 Subject: [PATCH] Allow listing languages, setting source language (#192) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/192 --- lib/pleroma/akkoma/translators/deepl.ex | 43 +++++++++++-- .../akkoma/translators/libre_translate.ex | 40 ++++++++++-- lib/pleroma/akkoma/translators/translator.ex | 4 +- .../controllers/translation_controller.ex | 43 +++++++++++++ .../api_spec/operations/status_operation.ex | 6 +- .../operations/translate_operation.ex | 41 ++++++++++++ .../controllers/status_controller.ex | 25 +++++--- lib/pleroma/web/router.ex | 5 ++ test/pleroma/translators/deepl_test.exs | 63 ++++++++++++++++++- .../translators/libre_translate_test.exs | 55 ++++++++++++++-- 10 files changed, 295 insertions(+), 30 deletions(-) create mode 100644 lib/pleroma/web/akkoma_api/controllers/translation_controller.ex create mode 100644 lib/pleroma/web/api_spec/operations/translate_operation.ex diff --git a/lib/pleroma/akkoma/translators/deepl.ex b/lib/pleroma/akkoma/translators/deepl.ex index 0a4a7fe10..f93fb7e59 100644 --- a/lib/pleroma/akkoma/translators/deepl.ex +++ b/lib/pleroma/akkoma/translators/deepl.ex @@ -22,8 +22,27 @@ defmodule Pleroma.Akkoma.Translators.DeepL do end @impl Pleroma.Akkoma.Translator - def translate(string, to_language) do - with {:ok, %{status: 200} = response} <- do_request(api_key(), tier(), string, to_language), + def languages do + with {:ok, %{status: 200} = response} <- do_languages(), + {:ok, body} <- Jason.decode(response.body) do + resp = + Enum.map(body, fn %{"language" => code, "name" => name} -> %{code: code, name: name} end) + + {:ok, resp} + else + {:ok, %{status: status} = response} -> + Logger.warning("DeepL: Request rejected: #{inspect(response)}") + {:error, "DeepL request failed (code #{status})"} + + {:error, reason} -> + {:error, reason} + end + end + + @impl Pleroma.Akkoma.Translator + def translate(string, from_language, to_language) do + with {:ok, %{status: 200} = response} <- + do_request(api_key(), tier(), string, from_language, to_language), {:ok, body} <- Jason.decode(response.body) do %{"translations" => [%{"text" => translated, "detected_source_language" => detected}]} = body @@ -39,14 +58,16 @@ defmodule Pleroma.Akkoma.Translators.DeepL do end end - defp do_request(api_key, tier, string, to_language) do + defp do_request(api_key, tier, string, from_language, to_language) do HTTP.post( base_url(tier) <> "translate", URI.encode_query( %{ text: string, - target_lang: to_language - }, + target_lang: to_language, + tag_handling: "html" + } + |> maybe_add_source(from_language), :rfc3986 ), [ @@ -55,4 +76,16 @@ defmodule Pleroma.Akkoma.Translators.DeepL do ] ) end + + defp maybe_add_source(opts, nil), do: opts + defp maybe_add_source(opts, lang), do: Map.put(opts, :source_lang, lang) + + defp do_languages() do + HTTP.get( + base_url(tier()) <> "languages?type=target", + [ + {"authorization", "DeepL-Auth-Key #{api_key()}"} + ] + ) + end end diff --git a/lib/pleroma/akkoma/translators/libre_translate.ex b/lib/pleroma/akkoma/translators/libre_translate.ex index 615d04192..319907c2f 100644 --- a/lib/pleroma/akkoma/translators/libre_translate.ex +++ b/lib/pleroma/akkoma/translators/libre_translate.ex @@ -14,10 +14,33 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do end @impl Pleroma.Akkoma.Translator - def translate(string, to_language) do - with {:ok, %{status: 200} = response} <- do_request(string, to_language), + def languages do + with {:ok, %{status: 200} = response} <- do_languages(), {:ok, body} <- Jason.decode(response.body) do - %{"translatedText" => translated, "detectedLanguage" => %{"language" => detected}} = body + resp = Enum.map(body, fn %{"code" => code, "name" => name} -> %{code: code, name: name} end) + {:ok, resp} + else + {:ok, %{status: status} = response} -> + Logger.warning("LibreTranslate: Request rejected: #{inspect(response)}") + {:error, "LibreTranslate request failed (code #{status})"} + + {:error, reason} -> + {:error, reason} + end + end + + @impl Pleroma.Akkoma.Translator + def translate(string, from_language, to_language) do + with {:ok, %{status: 200} = response} <- do_request(string, from_language, to_language), + {:ok, body} <- Jason.decode(response.body) do + %{"translatedText" => translated} = body + + detected = + if Map.has_key?(body, "detectedLanguage") do + get_in(body, ["detectedLanguage", "language"]) + else + from_language + end {:ok, detected, translated} else @@ -30,7 +53,7 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do end end - defp do_request(string, to_language) do + defp do_request(string, from_language, to_language) do url = URI.parse(url()) url = %{url | path: "/translate"} @@ -38,7 +61,7 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do to_string(url), Jason.encode!(%{ q: string, - source: "auto", + source: if(is_nil(from_language), do: "auto", else: from_language), target: to_language, format: "html", api_key: api_key() @@ -48,4 +71,11 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do ] ) end + + defp do_languages() do + url = URI.parse(url()) + url = %{url | path: "/languages"} + + HTTP.get(to_string(url)) + end end diff --git a/lib/pleroma/akkoma/translators/translator.ex b/lib/pleroma/akkoma/translators/translator.ex index 0276ed6c2..aa49b0655 100644 --- a/lib/pleroma/akkoma/translators/translator.ex +++ b/lib/pleroma/akkoma/translators/translator.ex @@ -1,3 +1,5 @@ defmodule Pleroma.Akkoma.Translator do - @callback translate(String.t(), String.t()) :: {:ok, String.t(), String.t()} | {:error, any()} + @callback translate(String.t(), String.t() | nil, String.t()) :: + {:ok, String.t(), String.t()} | {:error, any()} + @callback languages() :: {:ok, [%{name: String.t(), code: String.t()}]} | {:error, any()} end diff --git a/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex new file mode 100644 index 000000000..49ef89a50 --- /dev/null +++ b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex @@ -0,0 +1,43 @@ +defmodule Pleroma.Web.AkkomaAPI.TranslationController do + use Pleroma.Web, :controller + + alias Pleroma.Web.Plugs.OAuthScopesPlug + + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + + @unauthenticated_access %{fallback: :proceed_unauthenticated, scopes: []} + plug( + OAuthScopesPlug, + %{@unauthenticated_access | scopes: ["read:statuses"]} + when action in [ + :languages + ] + ) + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TranslationOperation + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + @doc "GET /api/v1/akkoma/translation/languages" + def languages(conn, _params) do + with {:ok, languages} <- get_languages() do + conn + |> json(languages) + else + e -> IO.inspect(e) + end + end + + defp get_languages do + module = Pleroma.Config.get([:translator, :module]) + + @cachex.fetch!(:translations_cache, "languages:#{module}}", fn _ -> + with {:ok, languages} <- module.languages() do + {:ok, languages} + else + {:error, err} -> {:ignore, {:error, err}} + end + end) + end +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 04a7bf5db..5332c9dca 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -413,7 +413,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do description: "View the translation of a given status", operationId: "StatusController.translation", security: [%{"oAuth" => ["read:statuses"]}], - parameters: [id_param(), language_param()], + parameters: [id_param(), language_param(), source_language_param()], responses: %{ 200 => Operation.response("Translation", "application/json", translation()), 400 => Operation.response("Error", "application/json", ApiError), @@ -572,6 +572,10 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do Operation.parameter(:language, :path, :string, "ISO 639 language code", example: "en") end + defp source_language_param do + Operation.parameter(:from, :query, :string, "ISO 639 language code", example: "en") + end + defp status_response do Operation.response("Status", "application/json", Status) end diff --git a/lib/pleroma/web/api_spec/operations/translate_operation.ex b/lib/pleroma/web/api_spec/operations/translate_operation.ex new file mode 100644 index 000000000..aa3b69a18 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/translate_operation.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.Web.ApiSpec.TranslationOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + @spec languages_operation() :: Operation.t() + def languages_operation() do + %Operation{ + tags: ["Retrieve status translation"], + summary: "Translate status", + description: "View the translation of a given status", + operationId: "AkkomaAPI.TranslationController.languages", + security: [%{"oAuth" => ["read:statuses"]}], + responses: %{ + 200 => Operation.response("Translation", "application/json", languages_schema()) + } + } + end + + defp languages_schema do + %Schema{ + type: "array", + items: %Schema{ + type: "object", + properties: %{ + code: %Schema{ + type: "string" + }, + name: %Schema{ + type: "string" + } + } + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index d9b93ca5e..41fbd7acf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -422,7 +422,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end @doc "GET /api/v1/statuses/:id/translations/:language" - def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) do + def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language} = params) do with {:enabled, true} <- {:enabled, Config.get([:translator, :enabled])}, %Activity{} = activity <- Activity.get_by_id_with_object(id), {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, @@ -431,6 +431,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do fetch_or_translate( activity.id, activity.object.data["content"], + Map.get(params, :from, nil), language, translation_module ) do @@ -449,16 +450,20 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end end - defp fetch_or_translate(status_id, text, language, translation_module) do - @cachex.fetch!(:user_cache, "translations:#{status_id}:#{language}", fn _ -> - value = translation_module.translate(text, language) - - with {:ok, _, _} <- value do - value - else - _ -> {:ignore, value} + defp fetch_or_translate(status_id, text, source_language, target_language, translation_module) do + @cachex.fetch!( + :translations_cache, + "translations:#{status_id}:#{source_language}:#{target_language}", + fn _ -> + value = translation_module.translate(text, source_language, target_language) + + with {:ok, _, _} <- value do + value + else + _ -> {:ignore, value} + end end - end) + ) end defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index aff7b67db..175b1c4c0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -462,6 +462,11 @@ defmodule Pleroma.Web.Router do put("/statuses/:id/emoji_reactions/:emoji", EmojiReactionController, :create) end + scope "/api/v1/akkoma", Pleroma.Web.AkkomaAPI do + pipe_through(:authenticated_api) + get("/translation/languages", TranslationController, :languages) + end + scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through(:authenticated_api) diff --git a/test/pleroma/translators/deepl_test.exs b/test/pleroma/translators/deepl_test.exs index 286d21d3e..58f23fe26 100644 --- a/test/pleroma/translators/deepl_test.exs +++ b/test/pleroma/translators/deepl_test.exs @@ -8,6 +8,36 @@ defmodule Pleroma.Akkoma.Translators.DeepLTest do clear_config([:deepl, :api_key], "deepl_api_key") end + test "should list supported languages" do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock(fn + %{method: :get, url: "https://api-free.deepl.com/v2/languages?type=target"} = env -> + auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end) + assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header + + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{ + "language" => "BG", + "name" => "Bulgarian", + "supports_formality" => false + }, + %{ + "language" => "CS", + "name" => "Czech", + "supports_formality" => false + } + ]) + } + end) + + assert {:ok, [%{code: "BG", name: "Bulgarian"}, %{code: "CS", name: "Czech"}]} = + DeepL.languages() + end + test "should work with the free tier" do clear_config([:deepl, :tier], :free) @@ -30,7 +60,7 @@ defmodule Pleroma.Akkoma.Translators.DeepLTest do } end) - assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end test "should work with the pro tier" do @@ -55,7 +85,33 @@ defmodule Pleroma.Akkoma.Translators.DeepLTest do } end) - assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") + end + + test "should assign source language if set" do + clear_config([:deepl, :tier], :pro) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://api.deepl.com/v2/translate"} = env -> + auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end) + assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header + assert String.contains?(env.body, "source_lang=ja") + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translations: [ + %{ + "text" => "I will crush you", + "detected_source_language" => "ja" + } + ] + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "ja", "en") end test "should gracefully fail if the API errors" do @@ -69,7 +125,8 @@ defmodule Pleroma.Akkoma.Translators.DeepLTest do } end) - assert {:error, "DeepL request failed (code 403)"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:error, "DeepL request failed (code 403)"} = + DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end end end diff --git a/test/pleroma/translators/libre_translate_test.exs b/test/pleroma/translators/libre_translate_test.exs index 9ed2c5323..d28d9278a 100644 --- a/test/pleroma/translators/libre_translate_test.exs +++ b/test/pleroma/translators/libre_translate_test.exs @@ -8,10 +8,35 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do clear_config([:libre_translate, :url], "http://libre.translate/translate") end + test "should list supported languages" do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock(fn + %{method: :get, url: "http://libre.translate/languages"} = _ -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{ + "code" => "en", + "name" => "English" + }, + %{ + "code" => "ar", + "name" => "Arabic" + } + ]) + } + end) + + assert {:ok, [%{code: "en", name: "English"}, %{code: "ar", name: "Arabic"}]} = + LibreTranslate.languages() + end + test "should work without an API key" do Tesla.Mock.mock(fn %{method: :post, url: "http://libre.translate/translate"} = env -> - assert {:ok, %{"api_key" => nil}} = Jason.decode(env.body) + assert {:ok, %{"api_key" => nil, "source" => "auto"}} = Jason.decode(env.body) %Tesla.Env{ status: 200, @@ -26,7 +51,8 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do } end) - assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end test "should work with an API key" do @@ -49,7 +75,8 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do } end) - assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end test "should gracefully handle API key errors" do @@ -67,7 +94,25 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do end) assert {:error, "libre_translate: request failed (code 403)"} = - LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") + end + + test "should set a source language if requested" do + Tesla.Mock.mock(fn + %{method: :post, url: "http://libre.translate/translate"} = env -> + assert {:ok, %{"api_key" => nil, "source" => "ja"}} = Jason.decode(env.body) + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translatedText: "I will crush you" + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "ja", "en") end test "should gracefully handle an unsupported language" do @@ -85,7 +130,7 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do end) assert {:error, "libre_translate: request failed (code 400)"} = - LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "zoop") + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "zoop") end end end -- 2.45.2