git.squeep.com Git - akkoma/blob - lib/mix/tasks/pleroma/uploads.ex

   1 defmodule Mix.Tasks.Pleroma.Uploads do
   2   use Mix.Task
   3   import Mix.Ecto
   4   alias Pleroma.{Upload, Uploaders.Local, Uploaders.S3}
   5   alias Mix.Tasks.Pleroma.Common
   6   require Logger
   7
   8   @log_every 50
   9   @shortdoc "Migrate uploads from local to remote storage"
  10   @longdioc """
  11    Manages uploads
  12    ## Migrate uploads from local to remote storage
  13
  14   """
  15
  16   def run(["migrate_local", target_uploader | args]) do
  17     delete? = Enum.member?(args, "--delete")
  18     Common.start_pleroma()
  19     local_path = Pleroma.Config.get!([Local, :uploads])
  20     uploader = Module.concat(Pleroma.Uploaders, target_uploader)
  21
  22     unless Code.ensure_loaded?(uploader) do
  23       raise("The uploader #{inspect(uploader)} is not an existing/loaded module.")
  24     end
  25
  26     target_enabled? = Pleroma.Config.get([Upload, :uploader]) == uploader
  27
  28     unless target_enabled? do
  29       Pleroma.Config.put([Upload, :uploader], uploader)
  30     end
  31
  32     Mix.shell().info("Migrating files from local #{local_path} to #{to_string(uploader)}")
  33
  34     if delete? do
  35       Mix.shell().info(
  36         "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
  37       )
  38
  39       :timer.sleep(:timer.seconds(5))
  40     end
  41
  42     uploads =
  43       File.ls!(local_path)
  44       |> Enum.map(fn id ->
  45         root_path = Path.join(local_path, id)
  46
  47         cond do
  48           File.dir?(root_path) ->
  49             files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])}
  50
  51             case List.first(files) do
  52               {id, file, path} ->
  53                 {%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path},
  54                  root_path}
  55
  56               _ ->
  57                 nil
  58             end
  59
  60           File.exists?(root_path) ->
  61             file = Path.basename(id)
  62             [hash, ext] = String.split(id, ".")
  63             {%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path}
  64
  65           true ->
  66             nil
  67         end
  68       end)
  69       |> Enum.filter(& &1)
  70
  71     total_count = length(uploads)
  72     Mix.shell().info("Found #{total_count} uploads")
  73
  74     uploads
  75     |> Task.async_stream(
  76       fn {upload, root_path} ->
  77         case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do
  78           {:ok, _} ->
  79             if delete?, do: File.rm_rf!(root_path)
  80             Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}")
  81             :ok
  82
  83           error ->
  84             Mix.shell().error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
  85         end
  86       end,
  87       timeout: 150_000
  88     )
  89     |> Stream.chunk_every(@log_every)
  90     |> Enum.reduce(0, fn done, count ->
  91       count = count + length(done)
  92       Mix.shell().info("Uploaded #{count}/#{total_count} files")
  93       count
  94     end)
  95
  96     Mix.shell().info("Done!")
  97   end
  98 end