git.squeep.com Git - akkoma/blob - lib/mix/tasks/pleroma/uploads.ex

   1 defmodule Mix.Tasks.Pleroma.Uploads do
   2   use Mix.Task
   3   import Mix.Ecto
   4   alias Pleroma.{Upload, Uploaders.Local, Uploaders.S3}
   5   alias Mix.Tasks.Pleroma.Common
   6   require Logger
   7
   8   @log_every 50
   9   @shortdoc "Migrates uploads from local to remote storage"
  10   @moduledoc """
  11    ## Migrate uploads from local to remote storage
  12        mix pleroma.uploads migrate_local TARGET_UPLOADER [OPTIONS...]
  13    Options:
  14    - `--delete` - delete local uploads after migrating them to the target uploader
  15
  16
  17    A list of avalible uploaders can be seen in config.exs
  18   """
  19
  20   def run(["migrate_local", target_uploader | args]) do
  21     delete? = Enum.member?(args, "--delete")
  22     Common.start_pleroma()
  23     local_path = Pleroma.Config.get!([Local, :uploads])
  24     uploader = Module.concat(Pleroma.Uploaders, target_uploader)
  25
  26     unless Code.ensure_loaded?(uploader) do
  27       raise("The uploader #{inspect(uploader)} is not an existing/loaded module.")
  28     end
  29
  30     target_enabled? = Pleroma.Config.get([Upload, :uploader]) == uploader
  31
  32     unless target_enabled? do
  33       Pleroma.Config.put([Upload, :uploader], uploader)
  34     end
  35
  36     Mix.shell().info("Migrating files from local #{local_path} to #{to_string(uploader)}")
  37
  38     if delete? do
  39       Mix.shell().info(
  40         "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)"
  41       )
  42
  43       :timer.sleep(:timer.seconds(5))
  44     end
  45
  46     uploads =
  47       File.ls!(local_path)
  48       |> Enum.map(fn id ->
  49         root_path = Path.join(local_path, id)
  50
  51         cond do
  52           File.dir?(root_path) ->
  53             files = for file <- File.ls!(root_path), do: {id, file, Path.join([root_path, file])}
  54
  55             case List.first(files) do
  56               {id, file, path} ->
  57                 {%Pleroma.Upload{id: id, name: file, path: id <> "/" <> file, tempfile: path},
  58                  root_path}
  59
  60               _ ->
  61                 nil
  62             end
  63
  64           File.exists?(root_path) ->
  65             file = Path.basename(id)
  66             [hash, ext] = String.split(id, ".")
  67             {%Pleroma.Upload{id: hash, name: file, path: file, tempfile: root_path}, root_path}
  68
  69           true ->
  70             nil
  71         end
  72       end)
  73       |> Enum.filter(& &1)
  74
  75     total_count = length(uploads)
  76     Mix.shell().info("Found #{total_count} uploads")
  77
  78     uploads
  79     |> Task.async_stream(
  80       fn {upload, root_path} ->
  81         case Upload.store(upload, uploader: uploader, filters: [], size_limit: nil) do
  82           {:ok, _} ->
  83             if delete?, do: File.rm_rf!(root_path)
  84             Logger.debug("uploaded: #{inspect(upload.path)} #{inspect(upload)}")
  85             :ok
  86
  87           error ->
  88             Mix.shell().error("failed to upload #{inspect(upload.path)}: #{inspect(error)}")
  89         end
  90       end,
  91       timeout: 150_000
  92     )
  93     |> Stream.chunk_every(@log_every)
  94     |> Enum.reduce(0, fn done, count ->
  95       count = count + length(done)
  96       Mix.shell().info("Uploaded #{count}/#{total_count} files")
  97       count
  98     end)
  99
 100     Mix.shell().info("Done!")
 101   end
 102 end