git.squeep.com Git - akkoma/blob - lib/pleroma/backup.ex

   1 # Pleroma: A lightweight social networking server
   2 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
   3 # SPDX-License-Identifier: AGPL-3.0-only
   4
   5 defmodule Pleroma.Backup do
   6   use Ecto.Schema
   7
   8   import Ecto.Changeset
   9   import Ecto.Query
  10
  11   alias Pleroma.Activity
  12   alias Pleroma.Bookmark
  13   alias Pleroma.Repo
  14   alias Pleroma.User
  15   alias Pleroma.Web.ActivityPub.ActivityPub
  16   alias Pleroma.Web.ActivityPub.Transmogrifier
  17   alias Pleroma.Web.ActivityPub.UserView
  18   alias Pleroma.Workers.BackupWorker
  19
  20   schema "backups" do
  21     field(:content_type, :string)
  22     field(:file_name, :string)
  23     field(:file_size, :integer, default: 0)
  24     field(:processed, :boolean, default: false)
  25
  26     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
  27
  28     timestamps()
  29   end
  30
  31   def create(user) do
  32     with :ok <- validate_limit(user),
  33          {:ok, backup} <- user |> new() |> Repo.insert() do
  34       BackupWorker.process(backup)
  35     end
  36   end
  37
  38   def new(user) do
  39     rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
  40     datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
  41     name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
  42
  43     %__MODULE__{
  44       user_id: user.id,
  45       content_type: "application/zip",
  46       file_name: name
  47     }
  48   end
  49
  50   def delete(backup) do
  51     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
  52
  53     with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
  54       Repo.delete(backup)
  55     end
  56   end
  57
  58   defp validate_limit(user) do
  59     case get_last(user.id) do
  60       %__MODULE__{inserted_at: inserted_at} ->
  61         days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
  62         diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
  63
  64         if diff > days do
  65           :ok
  66         else
  67           {:error, "Last export was less than #{days} days ago"}
  68         end
  69
  70       nil ->
  71         :ok
  72     end
  73   end
  74
  75   def get_last(user_id) do
  76     __MODULE__
  77     |> where(user_id: ^user_id)
  78     |> order_by(desc: :id)
  79     |> limit(1)
  80     |> Repo.one()
  81   end
  82
  83   def list(%User{id: user_id}) do
  84     __MODULE__
  85     |> where(user_id: ^user_id)
  86     |> order_by(desc: :id)
  87     |> Repo.all()
  88   end
  89
  90   def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
  91     __MODULE__
  92     |> where(user_id: ^user_id)
  93     |> where([b], b.id != ^latest_id)
  94     |> Repo.all()
  95     |> Enum.each(&BackupWorker.delete/1)
  96   end
  97
  98   def get(id), do: Repo.get(__MODULE__, id)
  99
 100   def process(%__MODULE__{} = backup) do
 101     with {:ok, zip_file} <- zip(backup),
 102          {:ok, %{size: size}} <- File.stat(zip_file),
 103          {:ok, _upload} <- upload(backup, zip_file) do
 104       backup
 105       |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
 106       |> Repo.update()
 107     end
 108   end
 109
 110   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
 111   def zip(%__MODULE__{} = backup) do
 112     backup = Repo.preload(backup, :user)
 113     name = String.trim_trailing(backup.file_name, ".zip")
 114     dir = Path.join(System.tmp_dir!(), name)
 115
 116     with :ok <- File.mkdir(dir),
 117          :ok <- actor(dir, backup.user),
 118          :ok <- statuses(dir, backup.user),
 119          :ok <- likes(dir, backup.user),
 120          :ok <- bookmarks(dir, backup.user),
 121          {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
 122          {:ok, _} <- File.rm_rf(dir) do
 123       {:ok, :binary.list_to_bin(zip_path)}
 124     end
 125   end
 126
 127   def upload(%__MODULE__{} = backup, zip_path) do
 128     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 129
 130     upload = %Pleroma.Upload{
 131       name: backup.file_name,
 132       tempfile: zip_path,
 133       content_type: backup.content_type,
 134       path: "backups/" <> backup.file_name
 135     }
 136
 137     with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
 138          :ok <- File.rm(zip_path) do
 139       {:ok, upload}
 140     end
 141   end
 142
 143   defp actor(dir, user) do
 144     with {:ok, json} <-
 145            UserView.render("user.json", %{user: user})
 146            |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
 147            |> Jason.encode() do
 148       File.write(dir <> "/actor.json", json)
 149     end
 150   end
 151
 152   defp write_header(file, name) do
 153     IO.write(
 154       file,
 155       """
 156       {
 157         "@context": "https://www.w3.org/ns/activitystreams",
 158         "id": "#{name}.json",
 159         "type": "OrderedCollection",
 160         "orderedItems": [
 161       """
 162     )
 163   end
 164
 165   defp write(query, dir, name, fun) do
 166     path = dir <> "/#{name}.json"
 167
 168     with {:ok, file} <- File.open(path, [:write, :utf8]),
 169          :ok <- write_header(file, name) do
 170       counter = :counters.new(1, [])
 171
 172       query
 173       |> Pleroma.RepoStreamer.chunk_stream(100)
 174       |> Stream.each(fn items ->
 175         Enum.each(items, fn i ->
 176           with {:ok, str} <- fun.(i),
 177                :ok <- IO.write(file, str <> ",\n") do
 178             :counters.add(counter, 1, 1)
 179           end
 180         end)
 181       end)
 182       |> Stream.run()
 183
 184       total = :counters.get(counter, 1)
 185
 186       with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
 187         File.close(file)
 188       end
 189     end
 190   end
 191
 192   defp bookmarks(dir, %{id: user_id} = _user) do
 193     Bookmark
 194     |> where(user_id: ^user_id)
 195     |> join(:inner, [b], activity in assoc(b, :activity))
 196     |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
 197     |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
 198   end
 199
 200   defp likes(dir, user) do
 201     user.ap_id
 202     |> Activity.Queries.by_actor()
 203     |> Activity.Queries.by_type("Like")
 204     |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
 205     |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
 206   end
 207
 208   defp statuses(dir, user) do
 209     opts =
 210       %{}
 211       |> Map.put(:type, ["Create", "Announce"])
 212       |> Map.put(:blocking_user, user)
 213       |> Map.put(:muting_user, user)
 214       |> Map.put(:reply_filtering_user, user)
 215       |> Map.put(:announce_filtering_user, user)
 216       |> Map.put(:user, user)
 217
 218     [[user.ap_id], User.following(user), Pleroma.List.memberships(user)]
 219     |> Enum.concat()
 220     |> ActivityPub.fetch_activities_query(opts)
 221     |> write(dir, "outbox", fn a ->
 222       with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
 223         activity |> Map.delete("@context") |> Jason.encode()
 224       end
 225     end)
 226   end
 227 end