Merge branch 'chore/remove-reference-to-distsn' into 'develop'
[akkoma] / lib / pleroma / emoji / pack.ex
1 defmodule Pleroma.Emoji.Pack do
2 @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
3 defstruct files: %{},
4 files_count: 0,
5 pack_file: nil,
6 path: nil,
7 pack: %{},
8 name: nil
9
10 @type t() :: %__MODULE__{
11 files: %{String.t() => Path.t()},
12 files_count: non_neg_integer(),
13 pack_file: Path.t(),
14 path: Path.t(),
15 pack: map(),
16 name: String.t()
17 }
18
19 alias Pleroma.Emoji
20
21 @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
22 def create(name) do
23 with :ok <- validate_not_empty([name]),
24 dir <- Path.join(emoji_path(), name),
25 :ok <- File.mkdir(dir) do
26 %__MODULE__{pack_file: Path.join(dir, "pack.json")}
27 |> save_pack()
28 end
29 end
30
31 defp paginate(entities, 1, page_size), do: Enum.take(entities, page_size)
32
33 defp paginate(entities, page, page_size) do
34 entities
35 |> Enum.chunk_every(page_size)
36 |> Enum.at(page - 1)
37 end
38
39 @spec show(keyword()) :: {:ok, t()} | {:error, atom()}
40 def show(opts) do
41 name = opts[:name]
42
43 with :ok <- validate_not_empty([name]),
44 {:ok, pack} <- load_pack(name) do
45 shortcodes =
46 pack.files
47 |> Map.keys()
48 |> Enum.sort()
49 |> paginate(opts[:page], opts[:page_size])
50
51 pack = Map.put(pack, :files, Map.take(pack.files, shortcodes))
52
53 {:ok, validate_pack(pack)}
54 end
55 end
56
57 @spec delete(String.t()) ::
58 {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
59 def delete(name) do
60 with :ok <- validate_not_empty([name]) do
61 emoji_path()
62 |> Path.join(name)
63 |> File.rm_rf()
64 end
65 end
66
67 @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
68 {:ok, t()} | {:error, File.posix() | atom()}
69 def add_file(name, shortcode, filename, file) do
70 with :ok <- validate_not_empty([name, shortcode, filename]),
71 :ok <- validate_emoji_not_exists(shortcode),
72 {:ok, pack} <- load_pack(name),
73 :ok <- save_file(file, pack, filename),
74 {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
75 Emoji.reload()
76 {:ok, updated_pack}
77 end
78 end
79
80 @spec delete_file(String.t(), String.t()) ::
81 {:ok, t()} | {:error, File.posix() | atom()}
82 def delete_file(name, shortcode) do
83 with :ok <- validate_not_empty([name, shortcode]),
84 {:ok, pack} <- load_pack(name),
85 :ok <- remove_file(pack, shortcode),
86 {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
87 Emoji.reload()
88 {:ok, updated_pack}
89 end
90 end
91
92 @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
93 {:ok, t()} | {:error, File.posix() | atom()}
94 def update_file(name, shortcode, new_shortcode, new_filename, force) do
95 with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
96 {:ok, pack} <- load_pack(name),
97 {:ok, filename} <- get_filename(pack, shortcode),
98 :ok <- validate_emoji_not_exists(new_shortcode, force),
99 :ok <- rename_file(pack, filename, new_filename),
100 {:ok, updated_pack} <-
101 pack
102 |> delete_emoji(shortcode)
103 |> put_emoji(new_shortcode, new_filename)
104 |> save_pack() do
105 Emoji.reload()
106 {:ok, updated_pack}
107 end
108 end
109
110 @spec import_from_filesystem() :: {:ok, [String.t()]} | {:error, File.posix() | atom()}
111 def import_from_filesystem do
112 emoji_path = emoji_path()
113
114 with {:ok, %{access: :read_write}} <- File.stat(emoji_path),
115 {:ok, results} <- File.ls(emoji_path) do
116 names =
117 results
118 |> Enum.map(&Path.join(emoji_path, &1))
119 |> Enum.reject(fn path ->
120 File.dir?(path) and File.exists?(Path.join(path, "pack.json"))
121 end)
122 |> Enum.map(&write_pack_contents/1)
123 |> Enum.reject(&is_nil/1)
124
125 {:ok, names}
126 else
127 {:ok, %{access: _}} -> {:error, :no_read_write}
128 e -> e
129 end
130 end
131
132 @spec list_remote(String.t()) :: {:ok, map()} | {:error, atom()}
133 def list_remote(url) do
134 uri = url |> String.trim() |> URI.parse()
135
136 with :ok <- validate_shareable_packs_available(uri) do
137 uri
138 |> URI.merge("/api/pleroma/emoji/packs")
139 |> http_get()
140 end
141 end
142
143 @spec list_local(keyword()) :: {:ok, map(), non_neg_integer()}
144 def list_local(opts) do
145 with {:ok, results} <- list_packs_dir() do
146 all_packs =
147 results
148 |> Enum.map(fn name ->
149 case load_pack(name) do
150 {:ok, pack} -> pack
151 _ -> nil
152 end
153 end)
154 |> Enum.reject(&is_nil/1)
155
156 packs =
157 all_packs
158 |> paginate(opts[:page], opts[:page_size])
159 |> Map.new(fn pack -> {pack.name, validate_pack(pack)} end)
160
161 {:ok, packs, length(all_packs)}
162 end
163 end
164
165 @spec get_archive(String.t()) :: {:ok, binary()} | {:error, atom()}
166 def get_archive(name) do
167 with {:ok, pack} <- load_pack(name),
168 :ok <- validate_downloadable(pack) do
169 {:ok, fetch_archive(pack)}
170 end
171 end
172
173 @spec download(String.t(), String.t(), String.t()) :: {:ok, t()} | {:error, atom()}
174 def download(name, url, as) do
175 uri = url |> String.trim() |> URI.parse()
176
177 with :ok <- validate_shareable_packs_available(uri),
178 {:ok, remote_pack} <- uri |> URI.merge("/api/pleroma/emoji/packs/#{name}") |> http_get(),
179 {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
180 {:ok, archive} <- download_archive(url, sha),
181 pack <- copy_as(remote_pack, as || name),
182 {:ok, _} = unzip(archive, pack_info, remote_pack, pack) do
183 # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
184 # in it to depend on itself
185 if pack_info[:fallback] do
186 save_pack(pack)
187 else
188 {:ok, pack}
189 end
190 end
191 end
192
193 @spec save_metadata(map(), t()) :: {:ok, t()} | {:error, File.posix()}
194 def save_metadata(metadata, %__MODULE__{} = pack) do
195 pack
196 |> Map.put(:pack, metadata)
197 |> save_pack()
198 end
199
200 @spec update_metadata(String.t(), map()) :: {:ok, t()} | {:error, File.posix()}
201 def update_metadata(name, data) do
202 with {:ok, pack} <- load_pack(name) do
203 if fallback_sha_changed?(pack, data) do
204 update_sha_and_save_metadata(pack, data)
205 else
206 save_metadata(data, pack)
207 end
208 end
209 end
210
211 @spec load_pack(String.t()) :: {:ok, t()} | {:error, :not_found}
212 def load_pack(name) do
213 pack_file = Path.join([emoji_path(), name, "pack.json"])
214
215 if File.exists?(pack_file) do
216 pack =
217 pack_file
218 |> File.read!()
219 |> from_json()
220 |> Map.put(:pack_file, pack_file)
221 |> Map.put(:path, Path.dirname(pack_file))
222 |> Map.put(:name, name)
223
224 files_count =
225 pack.files
226 |> Map.keys()
227 |> length()
228
229 {:ok, Map.put(pack, :files_count, files_count)}
230 else
231 {:error, :not_found}
232 end
233 end
234
235 @spec emoji_path() :: Path.t()
236 defp emoji_path do
237 [:instance, :static_dir]
238 |> Pleroma.Config.get!()
239 |> Path.join("emoji")
240 end
241
242 defp validate_emoji_not_exists(shortcode, force \\ false)
243 defp validate_emoji_not_exists(_shortcode, true), do: :ok
244
245 defp validate_emoji_not_exists(shortcode, _) do
246 case Emoji.get(shortcode) do
247 nil -> :ok
248 _ -> {:error, :already_exists}
249 end
250 end
251
252 defp write_pack_contents(path) do
253 pack = %__MODULE__{
254 files: files_from_path(path),
255 path: path,
256 pack_file: Path.join(path, "pack.json")
257 }
258
259 case save_pack(pack) do
260 {:ok, _pack} -> Path.basename(path)
261 _ -> nil
262 end
263 end
264
265 defp files_from_path(path) do
266 txt_path = Path.join(path, "emoji.txt")
267
268 if File.exists?(txt_path) do
269 # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
270 # Make a pack.json file from the contents of that emoji.txt file
271
272 # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
273
274 # Create a map of shortcodes to filenames from emoji.txt
275 txt_path
276 |> File.read!()
277 |> String.split("\n")
278 |> Enum.map(&String.trim/1)
279 |> Enum.map(fn line ->
280 case String.split(line, ~r/,\s*/) do
281 # This matches both strings with and without tags
282 # and we don't care about tags here
283 [name, file | _] ->
284 file_dir_name = Path.dirname(file)
285
286 if String.ends_with?(path, file_dir_name) do
287 {name, Path.basename(file)}
288 else
289 {name, file}
290 end
291
292 _ ->
293 nil
294 end
295 end)
296 |> Enum.reject(&is_nil/1)
297 |> Map.new()
298 else
299 # If there's no emoji.txt, assume all files
300 # that are of certain extensions from the config are emojis and import them all
301 pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
302 Emoji.Loader.make_shortcode_to_file_map(path, pack_extensions)
303 end
304 end
305
306 defp validate_pack(pack) do
307 info =
308 if downloadable?(pack) do
309 archive = fetch_archive(pack)
310 archive_sha = :crypto.hash(:sha256, archive) |> Base.encode16()
311
312 pack.pack
313 |> Map.put("can-download", true)
314 |> Map.put("download-sha256", archive_sha)
315 else
316 Map.put(pack.pack, "can-download", false)
317 end
318
319 Map.put(pack, :pack, info)
320 end
321
322 defp downloadable?(pack) do
323 # If the pack is set as shared, check if it can be downloaded
324 # That means that when asked, the pack can be packed and sent to the remote
325 # Otherwise, they'd have to download it from external-src
326 pack.pack["share-files"] &&
327 Enum.all?(pack.files, fn {_, file} ->
328 pack.path
329 |> Path.join(file)
330 |> File.exists?()
331 end)
332 end
333
334 defp create_archive_and_cache(pack, hash) do
335 files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)]
336
337 {:ok, {_, result}} =
338 :zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)])
339
340 ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
341 overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files))
342
343 Cachex.put!(
344 :emoji_packs_cache,
345 pack.name,
346 # if pack.json MD5 changes, the cache is not valid anymore
347 %{hash: hash, pack_data: result},
348 # Add a minute to cache time for every file in the pack
349 ttl: overall_ttl
350 )
351
352 result
353 end
354
355 defp save_pack(pack) do
356 with {:ok, json} <- Jason.encode(pack, pretty: true),
357 :ok <- File.write(pack.pack_file, json) do
358 {:ok, pack}
359 end
360 end
361
362 defp from_json(json) do
363 map = Jason.decode!(json)
364
365 struct(__MODULE__, %{files: map["files"], pack: map["pack"]})
366 end
367
368 defp validate_shareable_packs_available(uri) do
369 with {:ok, %{"links" => links}} <- uri |> URI.merge("/.well-known/nodeinfo") |> http_get(),
370 # Get the actual nodeinfo address and fetch it
371 {:ok, %{"metadata" => %{"features" => features}}} <-
372 links |> List.last() |> Map.get("href") |> http_get() do
373 if Enum.member?(features, "shareable_emoji_packs") do
374 :ok
375 else
376 {:error, :not_shareable}
377 end
378 end
379 end
380
381 defp validate_not_empty(list) do
382 if Enum.all?(list, fn i -> is_binary(i) and i != "" end) do
383 :ok
384 else
385 {:error, :empty_values}
386 end
387 end
388
389 defp save_file(file, pack, filename) do
390 file_path = Path.join(pack.path, filename)
391 create_subdirs(file_path)
392
393 case file do
394 %Plug.Upload{path: upload_path} ->
395 # Copy the uploaded file from the temporary directory
396 with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
397
398 url when is_binary(url) ->
399 # Download and write the file
400 file_contents = Tesla.get!(url).body
401 File.write(file_path, file_contents)
402 end
403 end
404
405 defp put_emoji(pack, shortcode, filename) do
406 files = Map.put(pack.files, shortcode, filename)
407 %{pack | files: files}
408 end
409
410 defp delete_emoji(pack, shortcode) do
411 files = Map.delete(pack.files, shortcode)
412 %{pack | files: files}
413 end
414
415 defp rename_file(pack, filename, new_filename) do
416 old_path = Path.join(pack.path, filename)
417 new_path = Path.join(pack.path, new_filename)
418 create_subdirs(new_path)
419
420 with :ok <- File.rename(old_path, new_path) do
421 remove_dir_if_empty(old_path, filename)
422 end
423 end
424
425 defp create_subdirs(file_path) do
426 if String.contains?(file_path, "/") do
427 file_path
428 |> Path.dirname()
429 |> File.mkdir_p!()
430 end
431 end
432
433 defp remove_file(pack, shortcode) do
434 with {:ok, filename} <- get_filename(pack, shortcode),
435 emoji <- Path.join(pack.path, filename),
436 :ok <- File.rm(emoji) do
437 remove_dir_if_empty(emoji, filename)
438 end
439 end
440
441 defp remove_dir_if_empty(emoji, filename) do
442 dir = Path.dirname(emoji)
443
444 if String.contains?(filename, "/") and File.ls!(dir) == [] do
445 File.rmdir!(dir)
446 else
447 :ok
448 end
449 end
450
451 defp get_filename(pack, shortcode) do
452 with %{^shortcode => filename} when is_binary(filename) <- pack.files,
453 true <- pack.path |> Path.join(filename) |> File.exists?() do
454 {:ok, filename}
455 else
456 _ -> {:error, :doesnt_exist}
457 end
458 end
459
460 defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
461
462 defp http_get(url) do
463 with {:ok, %{body: body}} <- url |> Pleroma.HTTP.get() do
464 Jason.decode(body)
465 end
466 end
467
468 defp list_packs_dir do
469 emoji_path = emoji_path()
470 # Create the directory first if it does not exist. This is probably the first request made
471 # with the API so it should be sufficient
472 with {:create_dir, :ok} <- {:create_dir, File.mkdir_p(emoji_path)},
473 {:ls, {:ok, results}} <- {:ls, File.ls(emoji_path)} do
474 {:ok, Enum.sort(results)}
475 else
476 {:create_dir, {:error, e}} -> {:error, :create_dir, e}
477 {:ls, {:error, e}} -> {:error, :ls, e}
478 end
479 end
480
481 defp validate_downloadable(pack) do
482 if downloadable?(pack), do: :ok, else: {:error, :cant_download}
483 end
484
485 defp copy_as(remote_pack, local_name) do
486 path = Path.join(emoji_path(), local_name)
487
488 %__MODULE__{
489 name: local_name,
490 path: path,
491 files: remote_pack["files"],
492 pack_file: Path.join(path, "pack.json")
493 }
494 end
495
496 defp unzip(archive, pack_info, remote_pack, local_pack) do
497 with :ok <- File.mkdir_p!(local_pack.path) do
498 files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end)
499 # Fallback cannot contain a pack.json file
500 files = if pack_info[:fallback], do: files, else: ['pack.json' | files]
501
502 :zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files)
503 end
504 end
505
506 defp fetch_pack_info(remote_pack, uri, name) do
507 case remote_pack["pack"] do
508 %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
509 {:ok,
510 %{
511 sha: sha,
512 url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
513 }}
514
515 %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
516 {:ok,
517 %{
518 sha: sha,
519 url: src,
520 fallback: true
521 }}
522
523 _ ->
524 {:error, "The pack was not set as shared and there is no fallback src to download from"}
525 end
526 end
527
528 defp download_archive(url, sha) do
529 with {:ok, %{body: archive}} <- Tesla.get(url) do
530 if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
531 {:ok, archive}
532 else
533 {:error, :invalid_checksum}
534 end
535 end
536 end
537
538 defp fetch_archive(pack) do
539 hash = :crypto.hash(:md5, File.read!(pack.pack_file))
540
541 case Cachex.get!(:emoji_packs_cache, pack.name) do
542 %{hash: ^hash, pack_data: archive} -> archive
543 _ -> create_archive_and_cache(pack, hash)
544 end
545 end
546
547 defp fallback_sha_changed?(pack, data) do
548 is_binary(data[:"fallback-src"]) and data[:"fallback-src"] != pack.pack["fallback-src"]
549 end
550
551 defp update_sha_and_save_metadata(pack, data) do
552 with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
553 :ok <- validate_has_all_files(pack, zip) do
554 fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
555
556 data
557 |> Map.put("fallback-src-sha256", fallback_sha)
558 |> save_metadata(pack)
559 end
560 end
561
562 defp validate_has_all_files(pack, zip) do
563 with {:ok, f_list} <- :zip.unzip(zip, [:memory]) do
564 # Check if all files from the pack.json are in the archive
565 pack.files
566 |> Enum.all?(fn {_, from_manifest} ->
567 List.keyfind(f_list, to_charlist(from_manifest), 0)
568 end)
569 |> if(do: :ok, else: {:error, :incomplete})
570 end
571 end
572 end