# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.Export do
+defmodule Pleroma.Backup do
+ use Ecto.Schema
+
+ import Ecto.Changeset
+ import Ecto.Query
+
alias Pleroma.Activity
alias Pleroma.Bookmark
+ alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
- import Ecto.Query
+ schema "backups" do
+ field(:content_type, :string)
+ field(:file_name, :string)
+ field(:file_size, :integer, default: 0)
+ field(:processed, :boolean, default: false)
- @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+ belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+ timestamps()
+ end
+
+ def create(user) do
+ with :ok <- validate_limit(user),
+ {:ok, backup} <- user |> new() |> Repo.insert() do
+ {:ok, backup}
+ end
+ end
+
+ def new(user) do
+ rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+ datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+ name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+ %__MODULE__{
+ user_id: user.id,
+ content_type: "application/zip",
+ file_name: name
+ }
+ end
+
+ defp validate_limit(user) do
+ case get_last(user.id) do
+ %__MODULE__{inserted_at: inserted_at} ->
+ days = 7
+ diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
- def run(user) do
- with {:ok, path} <- create_dir(user),
- :ok <- actor(path, user),
- :ok <- statuses(path, user),
- :ok <- likes(path, user),
- :ok <- bookmarks(path, user),
- {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
- {:ok, _} <- File.rm_rf(path) do
+ if diff > days do
+ :ok
+ else
+ {:error, "Last export was less than #{days} days ago"}
+ end
+
+ nil ->
+ :ok
+ end
+ end
+
+ def get_last(user_id) do
+ __MODULE__
+ |> where(user_id: ^user_id)
+ |> order_by(desc: :id)
+ |> limit(1)
+ |> Repo.one()
+ end
+
+ def process(%__MODULE__{} = backup) do
+ with {:ok, zip_file} <- zip(backup),
+ {:ok, %{size: size}} <- File.stat(zip_file),
+ {:ok, _upload} <- upload(backup, zip_file) do
+ backup
+ |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+ |> Repo.update()
+ end
+ end
+
+ @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+ def zip(%__MODULE__{} = backup) do
+ backup = Repo.preload(backup, :user)
+ name = String.trim_trailing(backup.file_name, ".zip")
+ dir = Path.join(System.tmp_dir!(), name)
+
+ with :ok <- File.mkdir(dir),
+ :ok <- actor(dir, backup.user),
+ :ok <- statuses(dir, backup.user),
+ :ok <- likes(dir, backup.user),
+ :ok <- bookmarks(dir, backup.user),
+ {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+ {:ok, _} <- File.rm_rf(dir) do
{:ok, :binary.list_to_bin(zip_path)}
end
end
- def upload(zip_path) do
+ def upload(%__MODULE__{} = backup, zip_path) do
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
- file_name = zip_path |> String.split("/") |> List.last()
- id = Ecto.UUID.generate()
upload = %Pleroma.Upload{
- id: id,
- name: file_name,
+ name: backup.file_name,
tempfile: zip_path,
- content_type: "application/zip",
- path: id <> "/" <> file_name
+ content_type: backup.content_type,
+ path: "backups/" <> backup.file_name
}
with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
end
end
- defp create_dir(user) do
- datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
- dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
-
- with :ok <- File.mkdir(dir), do: {:ok, dir}
- end
-
defp write_header(file, name) do
IO.write(
file,
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
-defmodule Pleroma.ExportTest do
+defmodule Pleroma.BackupTest do
use Pleroma.DataCase
import Pleroma.Factory
import Mock
+ alias Pleroma.Backup
alias Pleroma.Bookmark
alias Pleroma.Web.CommonAPI
- test "it exports user data" do
+ test "it creates a backup record" do
+ %{id: user_id} = user = insert(:user)
+ assert {:ok, backup} = Backup.create(user)
+
+ assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+ end
+
+ test "it return an error if the export limit is over" do
+ %{id: user_id} = user = insert(:user)
+ limit_days = 7
+
+ assert {:ok, backup} = Backup.create(user)
+ assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+
+ assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
+ end
+
+ test "it process a backup record" do
+ %{id: user_id} = user = insert(:user)
+ assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
+ assert {:ok, %Backup{} = backup} = Backup.process(backup)
+ assert backup.file_size > 0
+ assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+ end
+
+ test "it creates a zip archive with user data" do
user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
{:ok, %{object: %{data: %{"id" => id1}}} = status1} =
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
- assert {:ok, path} = Pleroma.Export.run(user)
+ assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+ assert {:ok, path} = Backup.zip(backup)
assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
File.rm!(path)
end
- describe "it uploads an exported backup archive" do
+ describe "it uploads a backup archive" do
setup do
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
Bookmark.create(user.id, status2.id)
Bookmark.create(user.id, status3.id)
- assert {:ok, path} = Pleroma.Export.run(user)
+ assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+ assert {:ok, path} = Backup.zip(backup)
- [path: path]
+ [path: path, backup: backup]
end
- test "S3", %{path: path} do
+ test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
with_mock ExAws, request: fn _ -> {:ok, :ok} end do
- assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+ assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
end
end
- test "Local", %{path: path} do
+ test "Local", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
- assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+ assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
end
end
end
projects / akkoma / commitdiff