Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop
[akkoma] / lib / pleroma / web / pleroma_api / controllers / emoji_api_controller.ex
1 defmodule Pleroma.Web.PleromaAPI.EmojiAPIController do
2 use Pleroma.Web, :controller
3
4 require Logger
5
6 @emoji_dir_path Path.join(
7 Pleroma.Config.get!([:instance, :static_dir]),
8 "emoji"
9 )
10
11 @cache_seconds_per_file Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file])
12
13 @doc """
14 Lists the packs available on the instance as JSON.
15
16 The information is public and does not require authentification. The format is
17 a map of "pack directory name" to pack.json contents.
18 """
19 def list_packs(conn, _params) do
20 with {:ok, results} <- File.ls(@emoji_dir_path) do
21 pack_infos =
22 results
23 |> Enum.filter(&has_pack_json?/1)
24 |> Enum.map(&load_pack/1)
25 # Check if all the files are in place and can be sent
26 |> Enum.map(&validate_pack/1)
27 # Transform into a map of pack-name => pack-data
28 |> Enum.into(%{})
29
30 json(conn, pack_infos)
31 end
32 end
33
34 defp has_pack_json?(file) do
35 dir_path = Path.join(@emoji_dir_path, file)
36 # Filter to only use the pack.json packs
37 File.dir?(dir_path) and File.exists?(Path.join(dir_path, "pack.json"))
38 end
39
40 defp load_pack(pack_name) do
41 pack_path = Path.join(@emoji_dir_path, pack_name)
42 pack_file = Path.join(pack_path, "pack.json")
43
44 {pack_name, Jason.decode!(File.read!(pack_file))}
45 end
46
47 defp validate_pack({name, pack}) do
48 pack_path = Path.join(@emoji_dir_path, name)
49
50 if can_download?(pack, pack_path) do
51 archive_for_sha = make_archive(name, pack, pack_path)
52 archive_sha = :crypto.hash(:sha256, archive_for_sha) |> Base.encode16()
53
54 pack =
55 pack
56 |> put_in(["pack", "can-download"], true)
57 |> put_in(["pack", "download-sha256"], archive_sha)
58
59 {name, pack}
60 else
61 {name, put_in(pack, ["pack", "can-download"], false)}
62 end
63 end
64
65 defp can_download?(pack, pack_path) do
66 # If the pack is set as shared, check if it can be downloaded
67 # That means that when asked, the pack can be packed and sent to the remote
68 # Otherwise, they'd have to download it from external-src
69 pack["pack"]["share-files"] &&
70 Enum.all?(pack["files"], fn {_, path} ->
71 File.exists?(Path.join(pack_path, path))
72 end)
73 end
74
75 defp create_archive_and_cache(name, pack, pack_dir, md5) do
76 files =
77 ['pack.json'] ++
78 (pack["files"] |> Enum.map(fn {_, path} -> to_charlist(path) end))
79
80 {:ok, {_, zip_result}} = :zip.zip('#{name}.zip', files, [:memory, cwd: to_charlist(pack_dir)])
81
82 cache_ms = :timer.seconds(@cache_seconds_per_file * Enum.count(files))
83
84 Cachex.put!(
85 :emoji_packs_cache,
86 name,
87 # if pack.json MD5 changes, the cache is not valid anymore
88 %{pack_json_md5: md5, pack_data: zip_result},
89 # Add a minute to cache time for every file in the pack
90 ttl: cache_ms
91 )
92
93 Logger.debug("Created an archive for the '#{name}' emoji pack, \
94 keeping it in cache for #{div(cache_ms, 1000)}s")
95
96 zip_result
97 end
98
99 defp make_archive(name, pack, pack_dir) do
100 # Having a different pack.json md5 invalidates cache
101 pack_file_md5 = :crypto.hash(:md5, File.read!(Path.join(pack_dir, "pack.json")))
102
103 case Cachex.get!(:emoji_packs_cache, name) do
104 %{pack_file_md5: ^pack_file_md5, pack_data: zip_result} ->
105 Logger.debug("Using cache for the '#{name}' shared emoji pack")
106 zip_result
107
108 _ ->
109 create_archive_and_cache(name, pack, pack_dir, pack_file_md5)
110 end
111 end
112
113 @doc """
114 An endpoint for other instances (via admin UI) or users (via browser)
115 to download packs that the instance shares.
116 """
117 def download_shared(conn, %{"name" => name}) do
118 pack_dir = Path.join(@emoji_dir_path, name)
119 pack_file = Path.join(pack_dir, "pack.json")
120
121 with {_, true} <- {:exists?, File.exists?(pack_file)},
122 pack = Jason.decode!(File.read!(pack_file)),
123 {_, true} <- {:can_download?, can_download?(pack, pack_dir)} do
124 zip_result = make_archive(name, pack, pack_dir)
125 send_download(conn, {:binary, zip_result}, filename: "#{name}.zip")
126 else
127 {:can_download?, _} ->
128 conn
129 |> put_status(:forbidden)
130 |> json(%{
131 error: "Pack #{name} cannot be downloaded from this instance, either pack sharing\
132 was disabled for this pack or some files are missing"
133 })
134
135 {:exists?, _} ->
136 conn
137 |> put_status(:not_found)
138 |> json(%{error: "Pack #{name} does not exist"})
139 end
140 end
141
142 @doc """
143 An admin endpoint to request downloading a pack named `pack_name` from the instance
144 `instance_address`.
145
146 If the requested instance's admin chose to share the pack, it will be downloaded
147 from that instance, otherwise it will be downloaded from the fallback source, if there is one.
148 """
149 def download_from(conn, %{"instance_address" => address, "pack_name" => name} = data) do
150 shareable_packs_available =
151 "#{address}/.well-known/nodeinfo"
152 |> Tesla.get!()
153 |> Map.get(:body)
154 |> Jason.decode!()
155 |> List.last()
156 |> Map.get("href")
157 # Get the actual nodeinfo address and fetch it
158 |> Tesla.get!()
159 |> Map.get(:body)
160 |> Jason.decode!()
161 |> get_in(["metadata", "features"])
162 |> Enum.member?("shareable_emoji_packs")
163
164 if shareable_packs_available do
165 full_pack =
166 "#{address}/api/pleroma/emoji/packs/list"
167 |> Tesla.get!()
168 |> Map.get(:body)
169 |> Jason.decode!()
170 |> Map.get(name)
171
172 pack_info_res =
173 case full_pack["pack"] do
174 %{"share-files" => true, "can-download" => true, "download-sha256" => sha} ->
175 {:ok,
176 %{
177 sha: sha,
178 uri: "#{address}/api/pleroma/emoji/packs/download_shared/#{name}"
179 }}
180
181 %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->
182 {:ok,
183 %{
184 sha: sha,
185 uri: src,
186 fallback: true
187 }}
188
189 _ ->
190 {:error,
191 "The pack was not set as shared and there is no fallback src to download from"}
192 end
193
194 with {:ok, %{sha: sha, uri: uri} = pinfo} <- pack_info_res,
195 %{body: emoji_archive} <- Tesla.get!(uri),
196 {_, true} <- {:checksum, Base.decode16!(sha) == :crypto.hash(:sha256, emoji_archive)} do
197 local_name = data["as"] || name
198 pack_dir = Path.join(@emoji_dir_path, local_name)
199 File.mkdir_p!(pack_dir)
200
201 files = Enum.map(full_pack["files"], fn {_, path} -> to_charlist(path) end)
202 # Fallback cannot contain a pack.json file
203 files = if pinfo[:fallback], do: files, else: ['pack.json'] ++ files
204
205 {:ok, _} = :zip.unzip(emoji_archive, cwd: to_charlist(pack_dir), file_list: files)
206
207 # Fallback can't contain a pack.json file, since that would cause the fallback-src-sha256
208 # in it to depend on itself
209 if pinfo[:fallback] do
210 pack_file_path = Path.join(pack_dir, "pack.json")
211
212 File.write!(pack_file_path, Jason.encode!(full_pack, pretty: true))
213 end
214
215 json(conn, "ok")
216 else
217 {:error, e} ->
218 conn |> put_status(:internal_server_error) |> json(%{error: e})
219
220 {:checksum, _} ->
221 conn
222 |> put_status(:internal_server_error)
223 |> json(%{error: "SHA256 for the pack doesn't match the one sent by the server"})
224 end
225 else
226 conn
227 |> put_status(:internal_server_error)
228 |> json(%{error: "The requested instance does not support sharing emoji packs"})
229 end
230 end
231
232 @doc """
233 Creates an empty pack named `name` which then can be updated via the admin UI.
234 """
235 def create(conn, %{"name" => name}) do
236 pack_dir = Path.join(@emoji_dir_path, name)
237
238 if not File.exists?(pack_dir) do
239 File.mkdir_p!(pack_dir)
240
241 pack_file_p = Path.join(pack_dir, "pack.json")
242
243 File.write!(
244 pack_file_p,
245 Jason.encode!(%{pack: %{}, files: %{}}, pretty: true)
246 )
247
248 conn |> json("ok")
249 else
250 conn
251 |> put_status(:conflict)
252 |> json(%{error: "A pack named \"#{name}\" already exists"})
253 end
254 end
255
256 @doc """
257 Deletes the pack `name` and all it's files.
258 """
259 def delete(conn, %{"name" => name}) do
260 pack_dir = Path.join(@emoji_dir_path, name)
261
262 case File.rm_rf(pack_dir) do
263 {:ok, _} ->
264 conn |> json("ok")
265
266 {:error, _} ->
267 conn
268 |> put_status(:internal_server_error)
269 |> json(%{error: "Couldn't delete the pack #{name}"})
270 end
271 end
272
273 @doc """
274 An endpoint to update `pack_names`'s metadata.
275
276 `new_data` is the new metadata for the pack, that will replace the old metadata.
277 """
278 def update_metadata(conn, %{"pack_name" => name, "new_data" => new_data}) do
279 pack_file_p = Path.join([@emoji_dir_path, name, "pack.json"])
280
281 full_pack = Jason.decode!(File.read!(pack_file_p))
282
283 # The new fallback-src is in the new data and it's not the same as it was in the old data
284 should_update_fb_sha =
285 not is_nil(new_data["fallback-src"]) and
286 new_data["fallback-src"] != full_pack["pack"]["fallback-src"]
287
288 with {_, true} <- {:should_update?, should_update_fb_sha},
289 %{body: pack_arch} <- Tesla.get!(new_data["fallback-src"]),
290 {:ok, flist} <- :zip.unzip(pack_arch, [:memory]),
291 {_, true} <- {:has_all_files?, has_all_files?(full_pack, flist)} do
292 fallback_sha = :crypto.hash(:sha256, pack_arch) |> Base.encode16()
293
294 new_data = Map.put(new_data, "fallback-src-sha256", fallback_sha)
295 update_metadata_and_send(conn, full_pack, new_data, pack_file_p)
296 else
297 {:should_update?, _} ->
298 update_metadata_and_send(conn, full_pack, new_data, pack_file_p)
299
300 {:has_all_files?, _} ->
301 conn
302 |> put_status(:bad_request)
303 |> json(%{error: "The fallback archive does not have all files specified in pack.json"})
304 end
305 end
306
307 # Check if all files from the pack.json are in the archive
308 defp has_all_files?(%{"files" => files}, flist) do
309 Enum.all?(files, fn {_, from_manifest} ->
310 Enum.find(flist, fn {from_archive, _} ->
311 to_string(from_archive) == from_manifest
312 end)
313 end)
314 end
315
316 defp update_metadata_and_send(conn, full_pack, new_data, pack_file_p) do
317 full_pack = Map.put(full_pack, "pack", new_data)
318 File.write!(pack_file_p, Jason.encode!(full_pack, pretty: true))
319
320 # Send new data back with fallback sha filled
321 json(conn, new_data)
322 end
323
324 defp get_filename(%{"filename" => filename}), do: filename
325
326 defp get_filename(%{"file" => file}) do
327 case file do
328 %Plug.Upload{filename: filename} -> filename
329 url when is_binary(url) -> Path.basename(url)
330 end
331 end
332
333 defp empty?(str), do: String.trim(str) == ""
334
335 defp update_file_and_send(conn, updated_full_pack, pack_file_p) do
336 # Write the emoji pack file
337 File.write!(pack_file_p, Jason.encode!(updated_full_pack, pretty: true))
338
339 # Return the modified file list
340 json(conn, updated_full_pack["files"])
341 end
342
343 @doc """
344 Updates a file in a pack.
345
346 Updating can mean three things:
347
348 - `add` adds an emoji named `shortcode` to the pack `pack_name`,
349 that means that the emoji file needs to be uploaded with the request
350 (thus requiring it to be a multipart request) and be named `file`.
351 There can also be an optional `filename` that will be the new emoji file name
352 (if it's not there, the name will be taken from the uploaded file).
353 - `update` changes emoji shortcode (from `shortcode` to `new_shortcode` or moves the file
354 (from the current filename to `new_filename`)
355 - `remove` removes the emoji named `shortcode` and it's associated file
356 """
357
358 # Add
359 def update_file(
360 conn,
361 %{"pack_name" => pack_name, "action" => "add", "shortcode" => shortcode} = params
362 ) do
363 pack_dir = Path.join(@emoji_dir_path, pack_name)
364 pack_file_p = Path.join(pack_dir, "pack.json")
365
366 full_pack = Jason.decode!(File.read!(pack_file_p))
367
368 with {_, false} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)},
369 filename <- get_filename(params),
370 false <- empty?(shortcode),
371 false <- empty?(filename) do
372 file_path = Path.join(pack_dir, filename)
373
374 # If the name contains directories, create them
375 if String.contains?(file_path, "/") do
376 File.mkdir_p!(Path.dirname(file_path))
377 end
378
379 case params["file"] do
380 %Plug.Upload{path: upload_path} ->
381 # Copy the uploaded file from the temporary directory
382 File.copy!(upload_path, file_path)
383
384 url when is_binary(url) ->
385 # Download and write the file
386 file_contents = Tesla.get!(url).body
387 File.write!(file_path, file_contents)
388 end
389
390 updated_full_pack = put_in(full_pack, ["files", shortcode], filename)
391 update_file_and_send(conn, updated_full_pack, pack_file_p)
392 else
393 {:has_shortcode, _} ->
394 conn
395 |> put_status(:conflict)
396 |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"})
397
398 true ->
399 conn
400 |> put_status(:bad_request)
401 |> json(%{error: "shortcode or filename cannot be empty"})
402 end
403 end
404
405 # Remove
406 def update_file(conn, %{
407 "pack_name" => pack_name,
408 "action" => "remove",
409 "shortcode" => shortcode
410 }) do
411 pack_dir = Path.join(@emoji_dir_path, pack_name)
412 pack_file_p = Path.join(pack_dir, "pack.json")
413
414 full_pack = Jason.decode!(File.read!(pack_file_p))
415
416 if Map.has_key?(full_pack["files"], shortcode) do
417 {emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode])
418
419 emoji_file_path = Path.join(pack_dir, emoji_file_path)
420
421 # Delete the emoji file
422 File.rm!(emoji_file_path)
423
424 # If the old directory has no more files, remove it
425 if String.contains?(emoji_file_path, "/") do
426 dir = Path.dirname(emoji_file_path)
427
428 if Enum.empty?(File.ls!(dir)) do
429 File.rmdir!(dir)
430 end
431 end
432
433 update_file_and_send(conn, updated_full_pack, pack_file_p)
434 else
435 conn
436 |> put_status(:bad_request)
437 |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
438 end
439 end
440
441 # Update
442 def update_file(
443 conn,
444 %{"pack_name" => pack_name, "action" => "update", "shortcode" => shortcode} = params
445 ) do
446 pack_dir = Path.join(@emoji_dir_path, pack_name)
447 pack_file_p = Path.join(pack_dir, "pack.json")
448
449 full_pack = Jason.decode!(File.read!(pack_file_p))
450
451 with {_, true} <- {:has_shortcode, Map.has_key?(full_pack["files"], shortcode)},
452 %{"new_shortcode" => new_shortcode, "new_filename" => new_filename} <- params,
453 false <- empty?(new_shortcode),
454 false <- empty?(new_filename) do
455 # First, remove the old shortcode, saving the old path
456 {old_emoji_file_path, updated_full_pack} = pop_in(full_pack, ["files", shortcode])
457 old_emoji_file_path = Path.join(pack_dir, old_emoji_file_path)
458 new_emoji_file_path = Path.join(pack_dir, new_filename)
459
460 # If the name contains directories, create them
461 if String.contains?(new_emoji_file_path, "/") do
462 File.mkdir_p!(Path.dirname(new_emoji_file_path))
463 end
464
465 # Move/Rename the old filename to a new filename
466 # These are probably on the same filesystem, so just rename should work
467 :ok = File.rename(old_emoji_file_path, new_emoji_file_path)
468
469 # If the old directory has no more files, remove it
470 if String.contains?(old_emoji_file_path, "/") do
471 dir = Path.dirname(old_emoji_file_path)
472
473 if Enum.empty?(File.ls!(dir)) do
474 File.rmdir!(dir)
475 end
476 end
477
478 # Then, put in the new shortcode with the new path
479 updated_full_pack = put_in(updated_full_pack, ["files", new_shortcode], new_filename)
480 update_file_and_send(conn, updated_full_pack, pack_file_p)
481 else
482 {:has_shortcode, _} ->
483 conn
484 |> put_status(:bad_request)
485 |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
486
487 true ->
488 conn
489 |> put_status(:bad_request)
490 |> json(%{error: "new_shortcode or new_filename cannot be empty"})
491
492 _ ->
493 conn
494 |> put_status(:bad_request)
495 |> json(%{error: "new_shortcode or new_file were not specified"})
496 end
497 end
498
499 def update_file(conn, %{"action" => action}) do
500 conn
501 |> put_status(:bad_request)
502 |> json(%{error: "Unknown action: #{action}"})
503 end
504
505 @doc """
506 Imports emoji from the filesystem.
507
508 Importing means checking all the directories in the
509 `$instance_static/emoji/` for directories which do not have
510 `pack.json`. If one has an emoji.txt file, that file will be used
511 to create a `pack.json` file with it's contents. If the directory has
512 neither, all the files with specific configured extenstions will be
513 assumed to be emojis and stored in the new `pack.json` file.
514 """
515 def import_from_fs(conn, _params) do
516 with {:ok, results} <- File.ls(@emoji_dir_path) do
517 imported_pack_names =
518 results
519 |> Enum.filter(fn file ->
520 dir_path = Path.join(@emoji_dir_path, file)
521 # Find the directories that do NOT have pack.json
522 File.dir?(dir_path) and not File.exists?(Path.join(dir_path, "pack.json"))
523 end)
524 |> Enum.map(&write_pack_json_contents/1)
525
526 json(conn, imported_pack_names)
527 else
528 {:error, _} ->
529 conn
530 |> put_status(:internal_server_error)
531 |> json(%{error: "Error accessing emoji pack directory"})
532 end
533 end
534
535 defp write_pack_json_contents(dir) do
536 dir_path = Path.join(@emoji_dir_path, dir)
537 emoji_txt_path = Path.join(dir_path, "emoji.txt")
538
539 files_for_pack = files_for_pack(emoji_txt_path, dir_path)
540 pack_json_contents = Jason.encode!(%{pack: %{}, files: files_for_pack})
541
542 File.write!(Path.join(dir_path, "pack.json"), pack_json_contents)
543
544 dir
545 end
546
547 defp files_for_pack(emoji_txt_path, dir_path) do
548 if File.exists?(emoji_txt_path) do
549 # There's an emoji.txt file, it's likely from a pack installed by the pack manager.
550 # Make a pack.json file from the contents of that emoji.txt fileh
551
552 # FIXME: Copy-pasted from Pleroma.Emoji/load_from_file_stream/2
553
554 # Create a map of shortcodes to filenames from emoji.txt
555 File.read!(emoji_txt_path)
556 |> String.split("\n")
557 |> Enum.map(&String.trim/1)
558 |> Enum.map(fn line ->
559 case String.split(line, ~r/,\s*/) do
560 # This matches both strings with and without tags
561 # and we don't care about tags here
562 [name, file | _] -> {name, file}
563 _ -> nil
564 end
565 end)
566 |> Enum.filter(fn x -> not is_nil(x) end)
567 |> Enum.into(%{})
568 else
569 # If there's no emoji.txt, assume all files
570 # that are of certain extensions from the config are emojis and import them all
571 pack_extensions = Pleroma.Config.get!([:emoji, :pack_extensions])
572 Pleroma.Emoji.Loader.make_shortcode_to_file_map(dir_path, pack_extensions)
573 end
574 end
575 end