Merge remote-tracking branch 'ilja/fix_tagpolicy_to_also_work_on_updates' into develop
[akkoma] / lib / pleroma / emoji / pack.ex
1 # Pleroma: A lightweight social networking server
2 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
3 # SPDX-License-Identifier: AGPL-3.0-only
4
5 defmodule Pleroma.Emoji.Pack do
6 @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
7 defstruct files: %{},
8 files_count: 0,
9 pack_file: nil,
10 path: nil,
11 pack: %{},
12 name: nil
13
14 @type t() :: %__MODULE__{
15 files: %{String.t() => Path.t()},
16 files_count: non_neg_integer(),
17 pack_file: Path.t(),
18 path: Path.t(),
19 pack: map(),
20 name: String.t()
21 }
22
23 @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
24
25 alias Pleroma.Emoji
26 alias Pleroma.Emoji.Pack
27 alias Pleroma.Utils
28
29 @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
30 def create(name) do
31 with :ok <- validate_not_empty([name]),
32 dir <- Path.join(emoji_path(), name),
33 :ok <- File.mkdir(dir) do
34 save_pack(%__MODULE__{pack_file: Path.join(dir, "pack.json")})
35 end
36 end
37
38 defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
39
40 defp paginate(entities, page, page_size) do
41 entities
42 |> Enum.chunk_every(page_size)
43 |> Enum.at(page - 1)
44 end
45
46 @spec show(keyword()) :: {:ok, t()} | {:error, atom()}
47 def show(opts) do
48 name = opts[:name]
49
50 with :ok <- validate_not_empty([name]),
51 {:ok, pack} <- load_pack(name) do
52 shortcodes =
53 pack.files
54 |> Map.keys()
55 |> Enum.sort()
56 |> paginate(opts[:page], opts[:page_size])
57
58 pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
59
60 {:ok, validate_pack(pack)}
61 end
62 end
63
64 @spec delete(String.t()) ::
65 {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
66 def delete(name) do
67 with :ok <- validate_not_empty([name]),
68 pack_path <- Path.join(emoji_path(), name) do
69 File.rm_rf(pack_path)
70 end
71 end
72
73 @spec unpack_zip_emojies(list(tuple())) :: list(map())
74 defp unpack_zip_emojies(zip_files) do
75 Enum.reduce(zip_files, [], fn
76 {_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
77 with(
78 filename <- Path.basename(path),
79 shortcode <- Path.basename(filename, Path.extname(filename)),
80 false <- Emoji.exist?(shortcode)
81 ) do
82 [%{path: path, filename: path, shortcode: shortcode} | acc]
83 else
84 _ -> acc
85 end
86
87 _, acc ->
88 acc
89 end)
90 end
91
92 @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
93 {:ok, t()}
94 | {:error, File.posix() | atom()}
95 def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
96 with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
97 [_ | _] = emojies <- unpack_zip_emojies(zip_files),
98 {:ok, tmp_dir} <- Utils.tmp_dir("emoji") do
99 try do
100 {:ok, _emoji_files} =
101 :zip.unzip(
102 to_charlist(file.path),
103 [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}]
104 )
105
106 {_, updated_pack} =
107 Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
108 emoji_file = %Plug.Upload{
109 filename: item[:filename],
110 path: Path.join(tmp_dir, item[:path])
111 }
112
113 {:ok, updated_pack} =
114 do_add_file(
115 emoji_pack,
116 item[:shortcode],
117 to_string(item[:filename]),
118 emoji_file
119 )
120
121 {item, updated_pack}
122 end)
123
124 Emoji.reload()
125
126 {:ok, updated_pack}
127 after
128 File.rm_rf(tmp_dir)
129 end
130 else
131 {:error, _} = error ->
132 error
133
134 _ ->
135 {:ok, pack}
136 end
137 end
138
139 def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
140 with :ok <- validate_not_empty([shortcode, filename]),
141 :ok <- validate_emoji_not_exists(shortcode),
142 {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
143 Emoji.reload()
144 {:ok, updated_pack}
145 end
146 end
147
148 defp do_add_file(pack, shortcode, filename, file) do
149 with :ok <- save_file(file, pack, filename) do
150 pack
151 |> put_emoji(shortcode, filename)
152 |> save_pack()
153 end
154 end
155
156 @spec delete_file(t(), String.t()) ::
157 {:ok, t()} | {:error, File.posix() | atom()}
158 def delete_file(%Pack{} = pack, shortcode) do
159 with :ok <- validate_not_empty([shortcode]),
160 :ok <- remove_file(pack, shortcode),
161 {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
162 Emoji.reload()
163 {:ok, updated_pack}
164 end
165 end
166
167 @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) ::
168 {:ok, t()} | {:error, File.posix() | atom()}
169 def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do
170 with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]),
171 {:ok, filename} <- get_filename(pack, shortcode),
172 :ok <- validate_emoji_not_exists(new_shortcode, force),
173 :ok <- rename_file(pack, filename, new_filename),
174 {:ok, updated_pack} <-
175 pack
176 |> delete_emoji(shortcode)
177 |> put_emoji(new_shortcode, new_filename)
178 |> save_pack() do
179 Emoji.reload()
180 {:ok, updated_pack}
181 end
182 end
183
184 @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, File.posix() | atom()}
185 def import_from_filesystem do
186 emoji_path = emoji_path()
187
188 with {:ok, %{access: :read_write}} <- File.stat(emoji_path),
189 {:ok, results} <- File.ls(emoji_path) do
190 names =
191 results
192 |> Enum.map(&Path.join(emoji_path, &1))
193 |> Enum.reject(fn path ->
194 File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
195 end)
196 |> Enum.map(&write_pack_contents/1)
197 |> Enum.reject(&is_nil/1)
198
199 {:ok, names}
200 else
201 {:ok, %{access: _}} -> {:error, :no_read_write}
202 e -> e
203 end
204 end
205
206 @spec list_remote(keyword()) :: {:ok, map()} | {:error, atom()}
207 def list_remote(opts) do
208 uri = opts[:url] |> String.trim() |> URI.parse()
209
210 with :ok <- validate_shareable_packs_available(uri) do
211 uri
212 |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}")
213 |> http_get()
214 end
215 end
216
217 @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
218 def list_local(opts) do
219 with {:ok, results} <- list_packs_dir() do
220 all_packs =
221 results
222 |> Enum.map(fn name ->
223 case load_pack(name) do
224 {:ok, pack} -> pack
225 _ -> nil
226 end
227 end)
228 |> Enum.reject(&is_nil/1)
229
230 packs =
231 all_packs
232 |> paginate(opts[:page], opts[:page_size])
233 |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
234
235 {:ok, packs, length(all_packs)}
236 end
237 end
238
239 @spec get_archive(String.t()) :: {:ok, binary()} | {:error, atom()}
240 def get_archive(name) do
241 with {:ok, pack} <- load_pack(name),
242 :ok <- validate_downloadable(pack) do
243 {:ok, fetch_archive(pack)}
244 end
245 end
246
247 @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
248 def download(name, url, as) do
249 uri = url |> String.trim() |> URI.parse()
250
251 with :ok <- validate_shareable_packs_available(uri),
252 {:ok, remote_pack} <-
253 uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(),
254 {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
255 {:ok, archive} <- download_archive(url, sha),
256 pack <- copy_as(remote_pack, as || name),
257 {:ok, _} = unzip(archive, pack_info, remote_pack, pack) do
258 # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
259 # in it to depend on itself
260 if pack_info[:fallback] do
261 save_pack(pack)
262 else
263 {:ok, pack}
264 end
265 end
266 end
267
268 @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
269 def save_metadata(metadata, %__MODULE__{} = pack) do
270 pack
271 |> Map.put(:pack, metadata)
272 |> save_pack()
273 end
274
275 @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
276 def update_metadata(name, data) do
277 with {:ok, pack} <- load_pack(name) do
278 if fallback_sha_changed?(pack, data) do
279 update_sha_and_save_metadata(pack, data)
280 else
281 save_metadata(data, pack)
282 end
283 end
284 end
285
286 @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()}
287 def load_pack(name) do
288 pack_file = Path.join([emoji_path(), name, "pack.json"])
289
290 with {:ok, _} <- File.stat(pack_file),
291 {:ok, pack_data} <- File.read(pack_file) do
292 pack =
293 from_json(
294 pack_data,
295 %{
296 pack_file: pack_file,
297 path: Path.dirname(pack_file),
298 name: name
299 }
300 )
301
302 files_count =
303 pack.files
304 |> Map.keys()
305 |> length()
306
307 {:ok, Map.put(pack, :files_count, files_count)}
308 end
309 end
310
311 @spec emoji_path() :: Path.t()
312 defp emoji_path do
313 [:instance, :static_dir]
314 |> Pleroma.Config.get!()
315 |> Path.join("emoji")
316 end
317
318 defp validate_emoji_not_exists(shortcode, force \\ false)
319 defp validate_emoji_not_exists(_shortcode, true), do: :ok
320
321 defp validate_emoji_not_exists(shortcode, _) do
322 if Emoji.exist?(shortcode) do
323 {:error, :already_exists}
324 else
325 :ok
326 end
327 end
328
329 defp write_pack_contents(path) do
330 pack = %__MODULE__{
331 files: files_from_path(path),
332 path: path,
333 pack_file: Path.join(path, "pack.json")
334 }
335
336 case save_pack(pack) do
337 {:ok, _pack} -> Path.basename(path)
338 _ -> nil
339 end
340 end
341
342 defp files_from_path(path) do
343 txt_path = Path.join(path, "emoji.txt")
344
345 if File.exists?(txt_path) do
346 # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
347 # Make a pack.json file from the contents of that emoji.txt file
348
349 # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
350
351 # Create a map of shortcodes to filenames from emoji.txt
352 txt_path
353 |> File.read!()
354 |> String.split("\n")
355 |> Enum.map(&String.trim/1)
356 |> Enum.map(fn line ->
357 case String.split(line, ~r/,\s*/) do
358 # This matches both strings with and without tags
359 # and we don't care about tags here
360 [name, file | _] ->
361 file_dir_name = Path.dirname(file)
362
363 if String.ends_with?(path, file_dir_name) do
364 {name, Path.basename(file)}
365 else
366 {name, file}
367 end
368
369 _ ->
370 nil
371 end
372 end)
373 |> Enum.reject(&is_nil/1)
374 |> Map.new()
375 else
376 # If there's no emoji.txt, assume all files
377 # that are of certain extensions from the config are emojis and import them all
378 pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
379 Emoji.Loader.make_shortcode_to_file_map(path, pack_extensions)
380 end
381 end
382
383 defp validate_pack(pack) do
384 info =
385 if downloadable?(pack) do
386 archive = fetch_archive(pack)
387 archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
388
389 pack.pack
390 |> Map.put("can-download", true)
391 |> Map.put("download-sha256", archive_sha)
392 else
393 Map.put(pack.pack, "can-download", false)
394 end
395
396 Map.put(pack, :pack, info)
397 end
398
399 defp downloadable?(pack) do
400 # If the pack is set as shared, check if it can be downloaded
401 # That means that when asked, the pack can be packed and sent to the remote
402 # Otherwise, they'd have to download it from external-src
403 pack.pack["share-files"] &&
404 Enum.all?(pack.files, fn {_, file} ->
405 pack.path
406 |> Path.join(file)
407 |> File.exists?()
408 end)
409 end
410
411 defp create_archive_and_cache(pack, hash) do
412 files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
413
414 {:ok, {_, result}} =
415 :zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
416
417 ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
418 overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
419
420 @cachex.put(
421 :emoji_packs_cache,
422 pack.name,
423 # if pack.json MD5 changes, the cache is not valid anymore
424 %{hash: hash, pack_data: result},
425 # Add a minute to cache time for every file in the pack
426 ttl: overall_ttl
427 )
428
429 result
430 end
431
432 defp save_pack(pack) do
433 with {:ok, json} <- Jason.encode(pack, pretty: true),
434 :ok <- File.write(pack.pack_file, json) do
435 {:ok, pack}
436 end
437 end
438
439 defp from_json(json, attrs) do
440 map = Jason.decode!(json)
441
442 pack_attrs =
443 attrs
444 |> Map.merge(%{
445 files: map["files"],
446 pack: map["pack"]
447 })
448
449 struct(__MODULE__, pack_attrs)
450 end
451
452 defp validate_shareable_packs_available(uri) do
453 with {:ok, %{"links" => links}} <- uri |> URI.merge("/.well-known/nodeinfo") |> http_get(),
454 # Get the actual nodeinfo address and fetch it
455 {:ok, %{"metadata" => %{"features" => features}}} <-
456 links |> List.last() |> Map.get("href") |> http_get() do
457 if Enum.member?(features, "shareable_emoji_packs") do
458 :ok
459 else
460 {:error, :not_shareable}
461 end
462 end
463 end
464
465 defp validate_not_empty(list) do
466 if Enum.all?(list, fn i -> is_binary(i) and i != "" end) do
467 :ok
468 else
469 {:error, :empty_values}
470 end
471 end
472
473 defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
474 file_path = Path.join(pack.path, filename)
475 create_subdirs(file_path)
476
477 with {:ok, _} <- File.copy(upload_path, file_path) do
478 :ok
479 end
480 end
481
482 defp put_emoji(pack, shortcode, filename) do
483 files = Map.put(pack.files, shortcode, filename)
484 %{pack | files: files, files_count: length(Map.keys(files))}
485 end
486
487 defp delete_emoji(pack, shortcode) do
488 files = Map.delete(pack.files, shortcode)
489 %{pack | files: files}
490 end
491
492 defp rename_file(pack, filename, new_filename) do
493 old_path = Path.join(pack.path, filename)
494 new_path = Path.join(pack.path, new_filename)
495 create_subdirs(new_path)
496
497 with :ok <- File.rename(old_path, new_path) do
498 remove_dir_if_empty(old_path, filename)
499 end
500 end
501
502 defp create_subdirs(file_path) do
503 with true <- String.contains?(file_path, "/"),
504 path <- Path.dirname(file_path),
505 false <- File.exists?(path) do
506 File.mkdir_p!(path)
507 end
508 end
509
510 defp remove_file(pack, shortcode) do
511 with {:ok, filename} <- get_filename(pack, shortcode),
512 emoji <- Path.join(pack.path, filename),
513 :ok <- File.rm(emoji) do
514 remove_dir_if_empty(emoji, filename)
515 end
516 end
517
518 defp remove_dir_if_empty(emoji, filename) do
519 dir = Path.dirname(emoji)
520
521 if String.contains?(filename, "/") and File.ls!(dir) == [] do
522 File.rmdir!(dir)
523 else
524 :ok
525 end
526 end
527
528 defp get_filename(pack, shortcode) do
529 with %{^shortcode => filename} when is_binary(filename) <- pack.files,
530 file_path <- Path.join(pack.path, filename),
531 {:ok, _} <- File.stat(file_path) do
532 {:ok, filename}
533 else
534 {:error, _} = error ->
535 error
536
537 _ ->
538 {:error, :doesnt_exist}
539 end
540 end
541
542 defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
543
544 defp http_get(url) do
545 with {:ok, %{body: body}} <- Pleroma.HTTP.get(url, [], []) do
546 Jason.decode(body)
547 end
548 end
549
550 defp list_packs_dir do
551 emoji_path = emoji_path()
552 # Create the directory first if it does not exist. This is probably the first request made
553 # with the API so it should be sufficient
554 with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
555 {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
556 {:ok, Enum.sort(results)}
557 else
558 {:create_dir, {:error, e}} -> {:error, :create_dir, e}
559 {:ls, {:error, e}} -> {:error, :ls, e}
560 end
561 end
562
563 defp validate_downloadable(pack) do
564 if downloadable?(pack), do: :ok, else: {:error, :cant_download}
565 end
566
567 defp copy_as(remote_pack, local_name) do
568 path = Path.join(emoji_path(), local_name)
569
570 %__MODULE__{
571 name: local_name,
572 path: path,
573 files: remote_pack["files"],
574 pack_file: Path.join(path, "pack.json")
575 }
576 end
577
578 defp unzip(archive, pack_info, remote_pack, local_pack) do
579 with :ok <- File.mkdir_p!(local_pack.path) do
580 files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
581 # Fallback cannot contain a pack.json file
582 files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
583
584 :zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
585 end
586 end
587
588 defp fetch_pack_info(remote_pack, uri, name) do
589 case remote_pack["pack"] do
590 %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
591 {:ok,
592 %{
593 sha: sha,
594 url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string()
595 }}
596
597 %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
598 {:ok,
599 %{
600 sha: sha,
601 url: src,
602 fallback: true
603 }}
604
605 _ ->
606 {:error, "The pack was not set as shared and there is no fallback src to download from"}
607 end
608 end
609
610 defp download_archive(url, sha) do
611 with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
612 if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
613 {:ok, archive}
614 else
615 {:error, :invalid_checksum}
616 end
617 end
618 end
619
620 defp fetch_archive(pack) do
621 hash = :crypto.hash(:md5, File.read!(pack.pack_file))
622
623 case @cachex.get!(:emoji_packs_cache, pack.name) do
624 %{hash: ^hash, pack_data: archive} -> archive
625 _ -> create_archive_and_cache(pack, hash)
626 end
627 end
628
629 defp fallback_sha_changed?(pack, data) do
630 is_binary(data[:"fallback-src"]) and data[:"fallback-src"] != pack.pack["fallback-src"]
631 end
632
633 defp update_sha_and_save_metadata(pack, data) do
634 with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
635 :ok <- validate_has_all_files(pack, zip) do
636 fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
637
638 data
639 |> Map.put("fallback-src-sha256", fallback_sha)
640 |> save_metadata(pack)
641 end
642 end
643
644 defp validate_has_all_files(pack, zip) do
645 with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
646 # Check if all files from the pack.json are in the archive
647 pack.files
648 |> Enum.all?(fn {_, from_manifest} ->
649 List.keyfind(f_list, to_charlist(from_manifest), 0)
650 end)
651 |> if(do: :ok, else: {:error, :incomplete})
652 end
653 end
654 end