git.squeep.com Git - akkoma/blob - lib/mix/tasks/pleroma/uploads.ex

   1 # Pleroma: A lightweight social networking server
   2 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
   3 # SPDX-License-Identifier: AGPL-3.0-only
   4
   5 defmodule Mix.Tasks.Pleroma.Uploads do
   6   use Mix.Task
   7   import Mix.Pleroma
   8   alias Pleroma.Upload
   9   alias Pleroma.Uploaders.Local
  10   require Logger
  11
  12   @log_every 50
  13
  14   @shortdoc "Migrates uploads from local to remote storage"
  15
  16   def run(["migrate_local", target_uploader | args]) do
  17     delete? = Enum.member?(args, "--delete")
  18     start_pleroma()
  19     local_path = Pleroma.Config.get!([Local, :uploads])
  20     uploader = Module.concat(Pleroma.Uploaders, target_uploader)
  21
  22     unless Code.ensure_loaded?(uploader) do
  23       raise("The uploader #{inspect(uploader)} is not an existing/loaded module.")
  24     end
  25
  26     target_enabled? = Pleroma.Config.get([Upload, :uploader]) == uploader
  27
  28     unless target_enabled? do
  29       Pleroma.Config.put([Upload, :uploader], uploader)
  30     end
  31
  32     shell_info("Migrating files from local #{local_path} to #{to_string(uploader)}")
  33
  34     if delete? do
  35       shell_info(
  36         "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
  37       )
  38
  39       :timer.sleep(:timer.seconds(5))
  40     end
  41
  42     uploads =
  43       File.ls!(local_path)
  44       |> Enum.map(fn id ->
  45         root_path = Path.join(local_path, id)
  46
  47         cond do
  48           File.dir?(root_path) ->
  49             files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])}
  50
  51             case List.first(files) do
  52               {id, file, path} ->
  53                 {%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path},
  54                  root_path}
  55
  56               _ ->
  57                 nil
  58             end
  59
  60           File.exists?(root_path) ->
  61             file = Path.basename(id)
  62             hash = Path.rootname(id)
  63             {%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path}
  64
  65           true ->
  66             nil
  67         end
  68       end)
  69       |> Enum.filter(& &1)
  70
  71     total_count = length(uploads)
  72     shell_info("Found #{total_count} uploads")
  73
  74     uploads
  75     |> Task.async_stream(
  76       fn {upload, root_path} ->
  77         case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do
  78           {:ok, _} ->
  79             if delete?, do: File.rm_rf!(root_path)
  80             Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}")
  81             :ok
  82
  83           error ->
  84             shell_error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
  85         end
  86       end,
  87       timeout: 150_000
  88     )
  89     |> Stream.chunk_every(@log_every)
  90     # credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation
  91     |> Enum.reduce(0, fn done, count ->
  92       count = count + length(done)
  93       shell_info("Uploaded #{count}/#{total_count} files")
  94       count
  95     end)
  96
  97     shell_info("Done!")
  98   end
  99 end