1 defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do
2 use Pleroma.Web, :controller
4 alias Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug
5 alias Pleroma.Plugs.OAuthScopesPlug
11 %{scopes: ["write"], admin: true}
24 [OAuthScopesPlug, ExpectPublicOrAuthenticatedCheckPlug]
25 when action in [:download_shared, :list_packs, :list_from]
28 defp emoji_dir_path do
30 Pleroma.Config.get!([:instance, :static_dir]),
36 Lists packs from the remote instance.
38 Since JS cannot ask remote instances for their packs due to CPS, it has to
41 def list_from(conn, %{"instance_address" => address}) do
42 address = String.trim(address)
44 if shareable_packs_available(address) do
46 "#{address}/api/pleroma/emoji/packs" |> Tesla.get!() |> Map.get(:body) |> Jason.decode!()
51 |> put_status(:internal_server_error)
52 |> json(%{error: "The requested instance does not support sharing emoji packs"})
57 Lists the packs available on the instance as JSON.
59 The information is public and does not require authentication. The format is
60 a map of "pack directory name" to pack.json contents.
62 def list_packs(conn, _params) do
63 # Create the directory first if it does not exist. This is probably the first request made
64 # with the API so it should be sufficient
65 with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_dir_path())},
66 {:ls, {:ok, results}} <- {:ls, File.ls(emoji_dir_path())} do
69 |> Enum.filter(&has_pack_json?/1)
70 |> Enum.map(&load_pack/1)
71 # Check if all the files are in place and can be sent
72 |> Enum.map(&validate_pack/1)
73 # Transform into a map of pack-name => pack-data
76 json(conn, pack_infos)
78 {:create_dir, {:error, e}} ->
80 |> put_status(:internal_server_error)
81 |> json(%{error: "Failed to create the emoji pack directory at #{emoji_dir_path()}: #{e}"})
85 |> put_status(:internal_server_error)
88 "Failed to get the contents of the emoji pack directory at #{emoji_dir_path()}: #{e}"
93 defp has_pack_json?(file) do
94 dir_path = Path.join(emoji_dir_path(), file)
95 # Filter to only use the pack.json packs
96 File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json"))
99 defp load_pack(pack_name) do
100 pack_path = Path.join(emoji_dir_path(), pack_name)
101 pack_file = Path.join(pack_path, "pack.json")
103 {pack_name, Jason.decode!(File.read!(pack_file))}
106 defp validate_pack({name, pack}) do
107 pack_path = Path.join(emoji_dir_path(), name)
109 if can_download?(pack, pack_path) do
110 archive_for_sha = make_archive(name, pack, pack_path)
111 archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16()
115 |> put_in(["pack", "can-download"], true)
116 |> put_in(["pack", "download-sha256"], archive_sha)
120 {name, put_in(pack, ["pack", "can-download"], false)}
124 defp can_download?(pack, pack_path) do
125 # If the pack is set as shared, check if it can be downloaded
126 # That means that when asked, the pack can be packed and sent to the remote
127 # Otherwise, they'd have to download it from external-src
128 pack["pack"]["share-files"] &&
129 Enum.all?(pack["files"], fn {_, path} ->
130 File.exists?(Path.join(pack_path, path))
134 defp create_archive_and_cache(name, pack, pack_dir, md5) do
137 (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end))
139 {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)])
141 cache_seconds_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
142 cache_ms = :timer.seconds(cache_seconds_per_file * Enum.count(files))
147 # if pack.json MD5 changes, the cache is not valid anymore
148 %{pack_json_md5: md5, pack_data: zip_result},
149 # Add a minute to cache time for every file in the pack
153 Logger.debug("Created an archive for the '#{name}' emoji pack, \
154 keeping it in cache for #{div(cache_ms, 1000)}s")
159 defp make_archive(name, pack, pack_dir) do
160 # Having a different pack.json md5 invalidates cache
161 pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json")))
163 case Cachex.get!(:emoji_packs_cache, name) do
164 %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} ->
165 Logger.debug("Using cache for the '#{name}' shared emoji pack")
169 create_archive_and_cache(name, pack, pack_dir, pack_file_md5)
174 An endpoint for other instances (via admin UI) or users (via browser)
175 to download packs that the instance shares.
177 def download_shared(conn, %{"name" => name}) do
178 pack_dir = Path.join(emoji_dir_path(), name)
179 pack_file = Path.join(pack_dir, "pack.json")
181 with {_, true} <- {:exists?, File.exists?(pack_file)},
182 pack = Jason.decode!(File.read!(pack_file)),
183 {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do
184 zip_result = make_archive(name, pack, pack_dir)
185 send_download(conn, {:binary, zip_result}, filename: "#{name}.zip")
187 {:can_download?, _} ->
189 |> put_status(:forbidden)
191 error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\
192 was disabled for this pack or some files are missing"
197 |> put_status(:not_found)
198 |> json(%{error: "Pack #{name} does not exist"})
202 defp shareable_packs_available(address) do
203 "#{address}/.well-known/nodeinfo"
210 # Get the actual nodeinfo address and fetch it
214 |> get_in(["metadata", "features"])
215 |> Enum.member?("shareable_emoji_packs")
219 An admin endpoint to request downloading and storing a pack named `pack_name` from the instance
222 If the requested instance's admin chose to share the pack, it will be downloaded
223 from that instance, otherwise it will be downloaded from the fallback source, if there is one.
225 def save_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do
226 address = String.trim(address)
228 if shareable_packs_available(address) do
230 "#{address}/api/pleroma/emoji/packs/list"
237 case full_pack["pack"] do
238 %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
242 uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}"
245 %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
255 "The pack was not set as shared and there is no fallback src to download from"}
258 with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res,
259 %{body: emoji_archive} <- Tesla.get!(uri),
260 {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do
261 local_name = data["as"] || name
262 pack_dir = Path.join(emoji_dir_path(), local_name)
263 File.mkdir_p!(pack_dir)
265 files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end)
266 # Fallback cannot contain a pack.json file
267 files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files
269 {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files)
271 # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
272 # in it to depend on itself
273 if pinfo[:fallback] do
274 pack_file_path = Path.join(pack_dir, "pack.json")
276 File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true))
282 conn |> put_status(:internal_server_error) |> json(%{error: e})
286 |> put_status(:internal_server_error)
287 |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"})
291 |> put_status(:internal_server_error)
292 |> json(%{error: "The requested instance does not support sharing emoji packs"})
297 Creates an empty pack named `name` which then can be updated via the admin UI.
299 def create(conn, %{"name" => name}) do
300 pack_dir = Path.join(emoji_dir_path(), name)
302 if not File.exists?(pack_dir) do
303 File.mkdir_p!(pack_dir)
305 pack_file_p = Path.join(pack_dir, "pack.json")
309 Jason.encode!(%{pack: %{}, files: %{}}, pretty: true)
315 |> put_status(:conflict)
316 |> json(%{error: "A pack named \"#{name}\" already exists"})
321 Deletes the pack `name` and all it's files.
323 def delete(conn, %{"name" => name}) do
324 pack_dir = Path.join(emoji_dir_path(), name)
326 case File.rm_rf(pack_dir) do
332 |> put_status(:internal_server_error)
333 |> json(%{error: "Couldn't delete the pack #{name}"})
338 An endpoint to update `pack_names`'s metadata.
340 `new_data` is the new metadata for the pack, that will replace the old metadata.
342 def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do
343 pack_file_p = Path.join([emoji_dir_path(), name, "pack.json"])
345 full_pack = Jason.decode!(File.read!(pack_file_p))
347 # The new fallback-src is in the new data and it's not the same as it was in the old data
348 should_update_fb_sha =
349 not is_nil(new_data["fallback-src"]) and
350 new_data["fallback-src"] != full_pack["pack"]["fallback-src"]
352 with {_, true} <- {:should_update?, should_update_fb_sha},
353 %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]),
354 {:ok, flist} <- :zip.unzip(pack_arch, [:memory]),
355 {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do
356 fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16()
358 new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha)
359 update_metadata_and_send(conn, full_pack, new_data, pack_file_p)
361 {:should_update?, _} ->
362 update_metadata_and_send(conn, full_pack, new_data, pack_file_p)
364 {:has_all_files?, _} ->
366 |> put_status(:bad_request)
367 |> json(%{error: "The fallback archive does not have all files specified in pack.json"})
371 # Check if all files from the pack.json are in the archive
372 defp has_all_files?(%{"files" => files}, flist) do
373 Enum.all?(files, fn {_, from_manifest} ->
374 Enum.find(flist, fn {from_archive, _} ->
375 to_string(from_archive) == from_manifest
380 defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do
381 full_pack = Map.put(full_pack, "pack", new_data)
382 File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true))
384 # Send new data back with fallback sha filled
388 defp get_filename(%Plug.Upload{filename: filename}), do: filename
389 defp get_filename(url) when is_binary(url), do: Path.basename(url)
391 defp empty?(str), do: String.trim(str) == ""
393 defp update_pack_file(updated_full_pack, pack_file_p) do
394 content = Jason.encode!(updated_full_pack, pretty: true)
396 File.write!(pack_file_p, content)
399 defp create_subdirs(file_path) do
400 if String.contains?(file_path, "/") do
407 defp pack_info(pack_name) do
408 dir = Path.join(emoji_dir_path(), pack_name)
409 json_path = Path.join(dir, "pack.json")
416 {dir, json_path, json}
420 Updates a file in a pack.
422 Updating can mean three things:
424 - `add` adds an emoji named `shortcode` to the pack `pack_name`,
425 that means that the emoji file needs to be uploaded with the request
426 (thus requiring it to be a multipart request) and be named `file`.
427 There can also be an optional `filename` that will be the new emoji file name
428 (if it's not there, the name will be taken from the uploaded file).
429 - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file
430 (from the current filename to `new_filename`)
431 - `remove` removes the emoji named `shortcode` and it's associated file
437 %{"pack_name" => pack_name, "action" => "add"} = params
440 if params["shortcode"] do
443 filename = get_filename(params["file"])
444 Path.basename(filename, Path.extname(filename))
447 {pack_dir, pack_file_p, full_pack} = pack_info(pack_name)
449 with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)},
450 filename <- params["filename"] || get_filename(params["file"]),
451 false <- empty?(shortcode),
452 false <- empty?(filename),
453 file_path <- Path.join(pack_dir, filename) do
454 # If the name contains directories, create them
455 create_subdirs(file_path)
457 case params["file"] do
458 %Plug.Upload{path: upload_path} ->
459 # Copy the uploaded file from the temporary directory
460 File.copy!(upload_path, file_path)
462 url when is_binary(url) ->
463 # Download and write the file
464 file_contents = Tesla.get!(url).body
465 File.write!(file_path, file_contents)
469 |> put_in(["files", shortcode], filename)
470 |> update_pack_file(pack_file_p)
472 json(conn, %{shortcode => filename})
474 {:has_shortcode, _} ->
476 |> put_status(:conflict)
477 |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"})
481 |> put_status(:bad_request)
482 |> json(%{error: "shortcode or filename cannot be empty"})
487 def update_file(conn, %{
488 "pack_name" => pack_name,
489 "action" => "remove",
490 "shortcode" => shortcode
492 {pack_dir, pack_file_p, full_pack} = pack_info(pack_name)
494 if Map.has_key?(full_pack["files"], shortcode) do
495 {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode])
497 emoji_file_path = Path.join(pack_dir, emoji_file_path)
499 # Delete the emoji file
500 File.rm!(emoji_file_path)
502 # If the old directory has no more files, remove it
503 if String.contains?(emoji_file_path, "/") do
504 dir = Path.dirname(emoji_file_path)
506 if Enum.empty?(File.ls!(dir)) do
511 update_pack_file(updated_full_pack, pack_file_p)
512 json(conn, %{shortcode => full_pack["files"][shortcode]})
515 |> put_status(:bad_request)
516 |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
523 %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params
525 {pack_dir, pack_file_p, full_pack} = pack_info(pack_name)
527 with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)},
528 %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params,
529 false <- empty?(new_shortcode),
530 false <- empty?(new_filename) do
531 # First, remove the old shortcode, saving the old path
532 {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode])
533 old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path)
534 new_emoji_file_path = Path.join(pack_dir, new_filename)
536 # If the name contains directories, create them
537 create_subdirs(new_emoji_file_path)
539 # Move/Rename the old filename to a new filename
540 # These are probably on the same filesystem, so just rename should work
541 :ok = File.rename(old_emoji_file_path, new_emoji_file_path)
543 # If the old directory has no more files, remove it
544 if String.contains?(old_emoji_file_path, "/") do
545 dir = Path.dirname(old_emoji_file_path)
547 if Enum.empty?(File.ls!(dir)) do
552 # Then, put in the new shortcode with the new path
554 |> put_in(["files", new_shortcode], new_filename)
555 |> update_pack_file(pack_file_p)
557 json(conn, %{new_shortcode => new_filename})
559 {:has_shortcode, _} ->
561 |> put_status(:bad_request)
562 |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
566 |> put_status(:bad_request)
567 |> json(%{error: "new_shortcode or new_filename cannot be empty"})
571 |> put_status(:bad_request)
572 |> json(%{error: "new_shortcode or new_file were not specified"})
576 def update_file(conn, %{"action" => action}) do
578 |> put_status(:bad_request)
579 |> json(%{error: "Unknown action: #{action}"})
583 Imports emoji from the filesystem.
585 Importing means checking all the directories in the
586 `$instance_static/emoji/` for directories which do not have
587 `pack.json`. If one has an emoji.txt file, that file will be used
588 to create a `pack.json` file with it's contents. If the directory has
589 neither, all the files with specific configured extenstions will be
590 assumed to be emojis and stored in the new `pack.json` file.
592 def import_from_fs(conn, _params) do
593 emoji_path = emoji_dir_path()
595 with {:ok, %{access: :read_write}} <- File.stat(emoji_path),
596 {:ok, results} <- File.ls(emoji_path) do
597 imported_pack_names =
599 |> Enum.filter(fn file ->
600 dir_path = Path.join(emoji_path, file)
601 # Find the directories that do NOT have pack.json
602 File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json"))
604 |> Enum.map(&write_pack_json_contents/1)
606 json(conn, imported_pack_names)
608 {:ok, %{access: _}} ->
610 |> put_status(:internal_server_error)
611 |> json(%{error: "Error: emoji pack directory must be writable"})
615 |> put_status(:internal_server_error)
616 |> json(%{error: "Error accessing emoji pack directory"})
620 defp write_pack_json_contents(dir) do
621 dir_path = Path.join(emoji_dir_path(), dir)
622 emoji_txt_path = Path.join(dir_path, "emoji.txt")
624 files_for_pack = files_for_pack(emoji_txt_path, dir_path)
625 pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack})
627 File.write!(Path.join(dir_path, "pack.json"), pack_json_contents)
632 defp files_for_pack(emoji_txt_path, dir_path) do
633 if File.exists?(emoji_txt_path) do
634 # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
635 # Make a pack.json file from the contents of that emoji.txt fileh
637 # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
639 # Create a map of shortcodes to filenames from emoji.txt
640 File.read!(emoji_txt_path)
641 |> String.split("\n")
642 |> Enum.map(&String.trim/1)
643 |> Enum.map(fn line ->
644 case String.split(line, ~r/,\s*/) do
645 # This matches both strings with and without tags
646 # and we don't care about tags here
647 [name, file | _] -> {name, file}
651 |> Enum.filter(fn x -> not is_nil(x) end)
654 # If there's no emoji.txt, assume all files
655 # that are of certain extensions from the config are emojis and import them all
656 pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
657 Pleroma.Emoji.Loader.make_shortcode_to_file_map(dir_path, pack_extensions)