From: Maksim Pechnikov Date: Sat, 22 Aug 2020 07:42:02 +0000 (+0300) Subject: upload emoji zip file X-Git-Url: https://git.squeep.com/?a=commitdiff_plain;h=f5845ff03395816902a637a28412f85e671575e7;p=akkoma upload emoji zip file --- diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index d076ae312..03aed33bb 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Emoji.Pack do } alias Pleroma.Emoji + alias Pleroma.Emoji.Pack @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values} def create(name) do @@ -64,24 +65,93 @@ defmodule Pleroma.Emoji.Pack do end end - @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) :: - {:ok, t()} | {:error, File.posix() | atom()} - def add_file(name, shortcode, filename, file) do - with :ok <- validate_not_empty([name, shortcode, filename]), + @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t()) :: + {:ok, t()} + | {:error, File.posix() | atom()} + def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do + with {:ok, zip_items} <- :zip.table(to_charlist(file.path)) do + emojies = + for {_, path, s, _, _, _} <- zip_items, elem(s, 2) == :regular do + filename = Path.basename(path) + shortcode = Path.basename(filename, Path.extname(filename)) + + %{ + path: path, + filename: path, + shortcode: shortcode, + exist: not is_nil(Pleroma.Emoji.get(shortcode)) + } + end + |> Enum.group_by(& &1[:exist]) + + case Map.get(emojies, false, []) do + [_ | _] = new_emojies -> + {:ok, tmp_dir} = Pleroma.Utils.tmp_dir("emoji") + + try do + {:ok, _emoji_files} = + :zip.unzip( + to_charlist(file.path), + [ + {:file_list, Enum.map(new_emojies, & &1[:path])}, + {:cwd, tmp_dir} + ] + ) + + {_, updated_pack} = + Enum.map_reduce(new_emojies, pack, fn item, emoji_pack -> + emoji_file = %Plug.Upload{ + filename: item[:filename], + path: Path.join(tmp_dir, item[:path]) + } + + {:ok, updated_pack} = + do_add_file( + emoji_pack, + item[:shortcode], + to_string(item[:filename]), + emoji_file + ) + + {item, updated_pack} + end) + + Emoji.reload() + + {:ok, updated_pack} + after + File.rm_rf(tmp_dir) + end + + _ -> + {:ok, pack} + end + end + end + + def add_file(%Pack{} = pack, shortcode, filename, file) do + with :ok <- validate_not_empty([shortcode, filename]), :ok <- validate_emoji_not_exists(shortcode), - {:ok, pack} <- load_pack(name), - :ok <- save_file(file, pack, filename), - {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do + {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do Emoji.reload() {:ok, updated_pack} end end - @spec delete_file(String.t(), String.t()) :: + defp do_add_file(pack, shortcode, filename, file) do + with :ok <- save_file(file, pack, filename), + {:ok, updated_pack} <- + pack + |> put_emoji(shortcode, filename) + |> save_pack() do + {:ok, updated_pack} + end + end + + @spec delete_file(t(), String.t()) :: {:ok, t()} | {:error, File.posix() | atom()} - def delete_file(name, shortcode) do - with :ok <- validate_not_empty([name, shortcode]), - {:ok, pack} <- load_pack(name), + def delete_file(%Pack{} = pack, shortcode) do + with :ok <- validate_not_empty([shortcode]), :ok <- remove_file(pack, shortcode), {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do Emoji.reload() @@ -89,11 +159,10 @@ defmodule Pleroma.Emoji.Pack do end end - @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) :: + @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) :: {:ok, t()} | {:error, File.posix() | atom()} - def update_file(name, shortcode, new_shortcode, new_filename, force) do - with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]), - {:ok, pack} <- load_pack(name), + def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do + with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]), {:ok, filename} <- get_filename(pack, shortcode), :ok <- validate_emoji_not_exists(new_shortcode, force), :ok <- rename_file(pack, filename, new_filename), @@ -386,19 +455,12 @@ defmodule Pleroma.Emoji.Pack do end end - defp save_file(file, pack, filename) do + defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do file_path = Path.join(pack.path, filename) create_subdirs(file_path) - case file do - %Plug.Upload{path: upload_path} -> - # Copy the uploaded file from the temporary directory - with {:ok, _} <- File.copy(upload_path, file_path), do: :ok - - url when is_binary(url) -> - # Download and write the file - file_contents = Tesla.get!(url).body - File.write(file_path, file_contents) + with {:ok, _} <- File.copy(upload_path, file_path) do + :ok end end diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 21d1159be..fcb8c64c7 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -24,4 +24,22 @@ defmodule Pleroma.Utils do def command_available?(command) do match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"])) end + + @doc "creates the uniq temporary directory" + @spec tmp_dir(String.t()) :: {:ok, String.t()} | {:error, :file.posix()} + def tmp_dir(prefix \\ "") do + sub_dir = [ + prefix, + Timex.to_unix(Timex.now()), + :os.getpid(), + String.downcase(Integer.to_string(:rand.uniform(0x100000000), 36)) + ] + + tmp_dir = Path.join(System.tmp_dir!(), Enum.join(sub_dir, "-")) + + case File.mkdir(tmp_dir) do + :ok -> {:ok, tmp_dir} + error -> error + end + end end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex index b6932157a..7dd4ce311 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex @@ -24,6 +24,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do parameters: [name_param()], responses: %{ 200 => Operation.response("Files Object", "application/json", files_object()), + 422 => Operation.response("Unprocessable Entity", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError), 400 => Operation.response("Bad Request", "application/json", ApiError), 409 => Operation.response("Conflict", "application/json", ApiError) } @@ -67,6 +69,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do parameters: [name_param()], responses: %{ 200 => Operation.response("Files Object", "application/json", files_object()), + 404 => Operation.response("Not Found", "application/json", ApiError), 400 => Operation.response("Bad Request", "application/json", ApiError), 409 => Operation.response("Conflict", "application/json", ApiError) } @@ -114,7 +117,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do ], responses: %{ 200 => Operation.response("Files Object", "application/json", files_object()), - 400 => Operation.response("Bad Request", "application/json", ApiError) + 400 => Operation.response("Bad Request", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) } } end diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex index ba9f07795..d10f46fde 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex @@ -22,7 +22,9 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do filename = params[:filename] || get_filename(params[:file]) shortcode = params[:shortcode] || Path.basename(filename, Path.extname(filename)) - with {:ok, pack} <- Pack.add_file(pack_name, shortcode, filename, params[:file]) do + with {:ok, pack} <- Pack.load_pack(pack_name), + {:ok, file} <- get_file(params[:file]), + {:ok, pack} <- Pack.add_file(pack, shortcode, filename, file) do json(conn, pack.files) else {:error, :already_exists} -> @@ -32,12 +34,12 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do {:error, :not_found} -> conn - |> put_status(:bad_request) + |> put_status(:not_found) |> json(%{error: "pack \"#{pack_name}\" is not found"}) {:error, :empty_values} -> conn - |> put_status(:bad_request) + |> put_status(:unprocessable_entity) |> json(%{error: "pack name, shortcode or filename cannot be empty"}) {:error, _} -> @@ -54,7 +56,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do new_filename = params[:new_filename] force = params[:force] - with {:ok, pack} <- Pack.update_file(pack_name, shortcode, new_shortcode, new_filename, force) do + with {:ok, pack} <- Pack.load_pack(pack_name), + {:ok, pack} <- Pack.update_file(pack, shortcode, new_shortcode, new_filename, force) do json(conn, pack.files) else {:error, :doesnt_exist} -> @@ -72,7 +75,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do {:error, :not_found} -> conn - |> put_status(:bad_request) + |> put_status(:not_found) |> json(%{error: "pack \"#{pack_name}\" is not found"}) {:error, :empty_values} -> @@ -90,7 +93,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do end def delete(conn, %{name: pack_name, shortcode: shortcode}) do - with {:ok, pack} <- Pack.delete_file(pack_name, shortcode) do + with {:ok, pack} <- Pack.load_pack(pack_name), + {:ok, pack} <- Pack.delete_file(pack, shortcode) do json(conn, pack.files) else {:error, :doesnt_exist} -> @@ -100,7 +104,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do {:error, :not_found} -> conn - |> put_status(:bad_request) + |> put_status(:not_found) |> json(%{error: "pack \"#{pack_name}\" is not found"}) {:error, :empty_values} -> @@ -119,4 +123,28 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileController do defp get_filename(%Plug.Upload{filename: filename}), do: filename defp get_filename(url) when is_binary(url), do: Path.basename(url) + + def get_file(%Plug.Upload{} = file), do: {:ok, file} + + def get_file(url) when is_binary(url) do + with {:ok, %Tesla.Env{body: body, status: code, headers: headers}} + when code in 200..299 <- Pleroma.HTTP.get(url) do + path = Plug.Upload.random_file!("emoji") + + content_type = + case List.keyfind(headers, "content-type", 0) do + {"content-type", value} -> value + nil -> nil + end + + File.write(path, body) + + {:ok, + %Plug.Upload{ + filename: Path.basename(url), + path: path, + content_type: content_type + }} + end + end end diff --git a/test/fixtures/finland-emojis.zip b/test/fixtures/finland-emojis.zip new file mode 100644 index 000000000..de7242ea1 Binary files /dev/null and b/test/fixtures/finland-emojis.zip differ diff --git a/test/web/pleroma_api/controllers/emoji_file_controller_test.exs b/test/web/pleroma_api/controllers/emoji_file_controller_test.exs index 56be130be..827a4c374 100644 --- a/test/web/pleroma_api/controllers/emoji_file_controller_test.exs +++ b/test/web/pleroma_api/controllers/emoji_file_controller_test.exs @@ -41,6 +41,45 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do :ok end + test "upload zip file with emojies", %{admin_conn: admin_conn} do + on_exit(fn -> + [ + "128px/a_trusted_friend-128.png", + "auroraborealis.png", + "1000px/baby_in_a_box.png", + "1000px/bear.png", + "128px/bear-128.png" + ] + |> Enum.each(fn path -> File.rm_rf!("#{@emoji_path}/test_pack/#{path}") end) + end) + + resp = + admin_conn + |> put_req_header("content-type", "multipart/form-data") + |> post("/api/pleroma/emoji/packs/test_pack/files", %{ + file: %Plug.Upload{ + content_type: "application/zip", + filename: "finland-emojis.zip", + path: Path.absname("test/fixtures/finland-emojis.zip") + } + }) + |> json_response_and_validate_schema(200) + + assert resp == %{ + "a_trusted_friend-128" => "128px/a_trusted_friend-128.png", + "auroraborealis" => "auroraborealis.png", + "baby_in_a_box" => "1000px/baby_in_a_box.png", + "bear" => "1000px/bear.png", + "bear-128" => "128px/bear-128.png", + "blank" => "blank.png", + "blank2" => "blank2.png" + } + + Enum.each(Map.values(resp), fn path -> + assert File.exists?("#{@emoji_path}/test_pack/#{path}") + end) + end + test "create shortcode exists", %{admin_conn: admin_conn} do assert admin_conn |> put_req_header("content-type", "multipart/form-data") @@ -140,7 +179,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do path: "#{@emoji_path}/test_pack/blank.png" } }) - |> json_response_and_validate_schema(:bad_request) == %{ + |> json_response_and_validate_schema(422) == %{ "error" => "pack name, shortcode or filename cannot be empty" } end @@ -156,7 +195,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do path: "#{@emoji_path}/test_pack/blank.png" } }) - |> json_response_and_validate_schema(:bad_request) == %{ + |> json_response_and_validate_schema(:not_found) == %{ "error" => "pack \"not_loaded\" is not found" } end @@ -164,7 +203,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do test "remove file with not loaded pack", %{admin_conn: admin_conn} do assert admin_conn |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=blank3") - |> json_response_and_validate_schema(:bad_request) == %{ + |> json_response_and_validate_schema(:not_found) == %{ "error" => "pack \"not_loaded\" is not found" } end @@ -172,8 +211,8 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do test "remove file with empty shortcode", %{admin_conn: admin_conn} do assert admin_conn |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=") - |> json_response_and_validate_schema(:bad_request) == %{ - "error" => "pack name or shortcode cannot be empty" + |> json_response_and_validate_schema(:not_found) == %{ + "error" => "pack \"not_loaded\" is not found" } end @@ -185,7 +224,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do new_shortcode: "blank3", new_filename: "dir_2/blank_3.png" }) - |> json_response_and_validate_schema(:bad_request) == %{ + |> json_response_and_validate_schema(:not_found) == %{ "error" => "pack \"not_loaded\" is not found" } end