1 # Pleroma: A lightweight social networking server
2 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
3 # SPDX-License-Identifier: AGPL-3.0-only
5 defmodule Pleroma.Emoji.Pack do
6 @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
14 @type t() :: %__MODULE__{
15 files: %{String.t() => Path.t()},
16 files_count: non_neg_integer(),
24 alias Pleroma.Emoji.Pack
27 @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
29 with :ok <- validate_not_empty([name]),
30 dir <- Path.join(emoji_path(), name),
31 :ok <- File.mkdir(dir) do
32 save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
36 defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
38 defp paginate(entities, page, page_size) do
40 |> Enum.chunk_every(page_size)
44 @spec show(keyword()) :: {:ok, t()} | {:error, atom()}
48 with :ok <- validate_not_empty([name]),
49 {:ok, pack} <- load_pack(name) do
54 |> paginate(opts[:page], opts[:page_size])
56 pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
58 {:ok, validate_pack(pack)}
62 @spec delete(String.t()) ::
63 {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
65 with :ok <- validate_not_empty([name]) do
72 @spec unpack_zip_emojies(list(tuple())) :: list(map())
73 defp unpack_zip_emojies(zip_files) do
74 Enum.reduce(zip_files, [], fn
75 {_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
77 filename <- Path.basename(path),
78 shortcode <- Path.basename(filename, Path.extname(filename)),
79 false <- Emoji.exist?(shortcode)
81 [%{path: path, filename: path, shortcode: shortcode} | acc]
91 @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
93 | {:error, File.posix() | atom()}
94 def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
95 with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
96 [_ | _] = emojies <- unpack_zip_emojies(zip_files),
97 {:ok, tmp_dir} <- Utils.tmp_dir("emoji") do
101 to_charlist(file.path),
102 [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}]
106 Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
107 emoji_file = %Plug.Upload{
108 filename: item[:filename],
109 path: Path.join(tmp_dir, item[:path])
112 {:ok, updated_pack} =
116 to_string(item[:filename]),
130 {:error, _} = error ->
138 def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
139 with :ok <- validate_not_empty([shortcode, filename]),
140 :ok <- validate_emoji_not_exists(shortcode),
141 {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
147 defp do_add_file(pack, shortcode, filename, file) do
148 with :ok <- save_file(file, pack, filename) do
150 |> put_emoji(shortcode, filename)
155 @spec delete_file(t(), String.t()) ::
156 {:ok, t()} | {:error, File.posix() | atom()}
157 def delete_file(%Pack{} = pack, shortcode) do
158 with :ok <- validate_not_empty([shortcode]),
159 :ok <- remove_file(pack, shortcode),
160 {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
166 @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) ::
167 {:ok, t()} | {:error, File.posix() | atom()}
168 def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do
169 with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]),
170 {:ok, filename} <- get_filename(pack, shortcode),
171 :ok <- validate_emoji_not_exists(new_shortcode, force),
172 :ok <- rename_file(pack, filename, new_filename),
173 {:ok, updated_pack} <-
175 |> delete_emoji(shortcode)
176 |> put_emoji(new_shortcode, new_filename)
183 @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, File.posix() | atom()}
184 def import_from_filesystem do
185 emoji_path = emoji_path()
187 with {:ok, %{access: :read_write}} <- File.stat(emoji_path),
188 {:ok, results} <- File.ls(emoji_path) do
191 |> Enum.map(&Path.join(emoji_path, &1))
192 |> Enum.reject(fn path ->
193 File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
195 |> Enum.map(&write_pack_contents/1)
196 |> Enum.reject(&is_nil/1)
200 {:ok, %{access: _}} -> {:error, :no_read_write}
205 @spec list_remote(keyword()) :: {:ok, map()} | {:error, atom()}
206 def list_remote(opts) do
207 uri = opts[:url] |> String.trim() |> URI.parse()
209 with :ok <- validate_shareable_packs_available(uri) do
211 |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}")
216 @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
217 def list_local(opts) do
218 with {:ok, results} <- list_packs_dir() do
221 |> Enum.map(fn name ->
222 case load_pack(name) do
227 |> Enum.reject(&is_nil/1)
231 |> paginate(opts[:page], opts[:page_size])
232 |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
234 {:ok, packs, length(all_packs)}
238 @spec get_archive(String.t()) :: {:ok, binary()} | {:error, atom()}
239 def get_archive(name) do
240 with {:ok, pack} <- load_pack(name),
241 :ok <- validate_downloadable(pack) do
242 {:ok, fetch_archive(pack)}
246 @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
247 def download(name, url, as) do
248 uri = url |> String.trim() |> URI.parse()
250 with :ok <- validate_shareable_packs_available(uri),
251 {:ok, remote_pack} <-
252 uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(),
253 {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
254 {:ok, archive} <- download_archive(url, sha),
255 pack <- copy_as(remote_pack, as || name),
256 {:ok, _} = unzip(archive, pack_info, remote_pack, pack) do
257 # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
258 # in it to depend on itself
259 if pack_info[:fallback] do
267 @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
268 def save_metadata(metadata, %__MODULE__{} = pack) do
270 |> Map.put(:pack, metadata)
274 @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
275 def update_metadata(name, data) do
276 with {:ok, pack} <- load_pack(name) do
277 if fallback_sha_changed?(pack, data) do
278 update_sha_and_save_metadata(pack, data)
280 save_metadata(data, pack)
285 @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
286 def load_pack(name) do
287 pack_file = Path.join([emoji_path(), name, "pack.json"])
289 with {:ok, _} <- File.stat(pack_file),
290 {:ok, pack_data} <- File.read(pack_file) do
295 pack_file: pack_file,
296 path: Path.dirname(pack_file),
306 {:ok, Map.put(pack, :files_count, files_count)}
310 @spec emoji_path() :: Path.t()
312 [:instance, :static_dir]
313 |> Pleroma.Config.get!()
314 |> Path.join("emoji")
317 defp validate_emoji_not_exists(shortcode, force \\ false)
318 defp validate_emoji_not_exists(_shortcode, true), do: :ok
320 defp validate_emoji_not_exists(shortcode, _) do
321 if Emoji.exist?(shortcode) do
322 {:error, :already_exists}
328 defp write_pack_contents(path) do
330 files: files_from_path(path),
332 pack_file: Path.join(path, "pack.json")
335 case save_pack(pack) do
336 {:ok, _pack} -> Path.basename(path)
341 defp files_from_path(path) do
342 txt_path = Path.join(path, "emoji.txt")
344 if File.exists?(txt_path) do
345 # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
346 # Make a pack.json file from the contents of that emoji.txt file
348 # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
350 # Create a map of shortcodes to filenames from emoji.txt
353 |> String.split("\n")
354 |> Enum.map(&String.trim/1)
355 |> Enum.map(fn line ->
356 case String.split(line, ~r/,\s*/) do
357 # This matches both strings with and without tags
358 # and we don't care about tags here
360 file_dir_name = Path.dirname(file)
362 if String.ends_with?(path, file_dir_name) do
363 {name, Path.basename(file)}
372 |> Enum.reject(&is_nil/1)
375 # If there's no emoji.txt, assume all files
376 # that are of certain extensions from the config are emojis and import them all
377 pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
378 Emoji.Loader.make_shortcode_to_file_map(path, pack_extensions)
382 defp validate_pack(pack) do
384 if downloadable?(pack) do
385 archive = fetch_archive(pack)
386 archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
389 |> Map.put("can-download", true)
390 |> Map.put("download-sha256", archive_sha)
392 Map.put(pack.pack, "can-download", false)
395 Map.put(pack, :pack, info)
398 defp downloadable?(pack) do
399 # If the pack is set as shared, check if it can be downloaded
400 # That means that when asked, the pack can be packed and sent to the remote
401 # Otherwise, they'd have to download it from external-src
402 pack.pack["share-files"] &&
403 Enum.all?(pack.files, fn {_, file} ->
410 defp create_archive_and_cache(pack, hash) do
411 files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
414 :zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
416 ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
417 overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
422 # if pack.json MD5 changes, the cache is not valid anymore
423 %{hash: hash, pack_data: result},
424 # Add a minute to cache time for every file in the pack
431 defp save_pack(pack) do
432 with {:ok, json} <- Jason.encode(pack, pretty: true),
433 :ok <- File.write(pack.pack_file, json) do
438 defp from_json(json, attrs) do
439 map = Jason.decode!(json)
448 struct(__MODULE__, pack_attrs)
451 defp validate_shareable_packs_available(uri) do
452 with {:ok, %{"links" => links}} <- uri |> URI.merge("/.well-known/nodeinfo") |> http_get(),
453 # Get the actual nodeinfo address and fetch it
454 {:ok, %{"metadata" => %{"features" => features}}} <-
455 links |> List.last() |> Map.get("href") |> http_get() do
456 if Enum.member?(features, "shareable_emoji_packs") do
459 {:error, :not_shareable}
464 defp validate_not_empty(list) do
465 if Enum.all?(list, fn i -> is_binary(i) and i != "" end) do
468 {:error, :empty_values}
472 defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
473 file_path = Path.join(pack.path, filename)
474 create_subdirs(file_path)
476 with {:ok, _} <- File.copy(upload_path, file_path) do
481 defp put_emoji(pack, shortcode, filename) do
482 files = Map.put(pack.files, shortcode, filename)
483 %{pack | files: files, files_count: length(Map.keys(files))}
486 defp delete_emoji(pack, shortcode) do
487 files = Map.delete(pack.files, shortcode)
488 %{pack | files: files}
491 defp rename_file(pack, filename, new_filename) do
492 old_path = Path.join(pack.path, filename)
493 new_path = Path.join(pack.path, new_filename)
494 create_subdirs(new_path)
496 with :ok <- File.rename(old_path, new_path) do
497 remove_dir_if_empty(old_path, filename)
501 defp create_subdirs(file_path) do
502 with true <- String.contains?(file_path, "/"),
503 path <- Path.dirname(file_path),
504 false <- File.exists?(path) do
509 defp remove_file(pack, shortcode) do
510 with {:ok, filename} <- get_filename(pack, shortcode),
511 emoji <- Path.join(pack.path, filename),
512 :ok <- File.rm(emoji) do
513 remove_dir_if_empty(emoji, filename)
517 defp remove_dir_if_empty(emoji, filename) do
518 dir = Path.dirname(emoji)
520 if String.contains?(filename, "/") and File.ls!(dir) == [] do
527 defp get_filename(pack, shortcode) do
528 with %{^shortcode => filename} when is_binary(filename) <- pack.files,
529 file_path <- Path.join(pack.path, filename),
530 {:ok, _} <- File.stat(file_path) do
533 {:error, _} = error ->
537 {:error, :doesnt_exist}
541 defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
543 defp http_get(url) do
544 with {:ok, %{body: body}} <- Pleroma.HTTP.get(url, [], pool: :default) do
549 defp list_packs_dir do
550 emoji_path = emoji_path()
551 # Create the directory first if it does not exist. This is probably the first request made
552 # with the API so it should be sufficient
553 with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
554 {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
555 {:ok, Enum.sort(results)}
557 {:create_dir, {:error, e}} -> {:error, :create_dir, e}
558 {:ls, {:error, e}} -> {:error, :ls, e}
562 defp validate_downloadable(pack) do
563 if downloadable?(pack), do: :ok, else: {:error, :cant_download}
566 defp copy_as(remote_pack, local_name) do
567 path = Path.join(emoji_path(), local_name)
572 files: remote_pack["files"],
573 pack_file: Path.join(path, "pack.json")
577 defp unzip(archive, pack_info, remote_pack, local_pack) do
578 with :ok <- File.mkdir_p!(local_pack.path) do
579 files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
580 # Fallback cannot contain a pack.json file
581 files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
583 :zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
587 defp fetch_pack_info(remote_pack, uri, name) do
588 case remote_pack["pack"] do
589 %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
593 url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string()
596 %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
605 {:error, "The pack was not set as shared and there is no fallback src to download from"}
609 defp download_archive(url, sha) do
610 with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
611 if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
614 {:error, :invalid_checksum}
619 defp fetch_archive(pack) do
620 hash = :crypto.hash(:md5, File.read!(pack.pack_file))
622 case Cachex.get!(:emoji_packs_cache, pack.name) do
623 %{hash: ^hash, pack_data: archive} -> archive
624 _ -> create_archive_and_cache(pack, hash)
628 defp fallback_sha_changed?(pack, data) do
629 is_binary(data[:"fallback-src"]) and data[:"fallback-src"] != pack.pack["fallback-src"]
632 defp update_sha_and_save_metadata(pack, data) do
633 with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
634 :ok <- validate_has_all_files(pack, zip) do
635 fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
638 |> Map.put("fallback-src-sha256", fallback_sha)
639 |> save_metadata(pack)
643 defp validate_has_all_files(pack, zip) do
644 with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
645 # Check if all files from the pack.json are in the archive
647 |> Enum.all?(fn {_, from_manifest} ->
648 List.keyfind(f_list, to_charlist(from_manifest), 0)
650 |> if(do: :ok, else: {:error, :incomplete})