Merge develop

diff --combined CHANGELOG.md
index f2c9e106ef692853a048294b08dbb2c0c449747c,4b7fb603dbc3e04695fc7a84c64eab0525de060d..b385264f19555895d2cc83eb5b4a64fcb4961962
--- 1/CHANGELOG.md
--- 2/CHANGELOG.md
+++ b/CHANGELOG.md
@@@ -8,18 -8,9 +8,18 @@@ The format is based on [Keep a Changelo
  ### Changed
  <details>
    <summary>API Changes</summary>
 +
  - **Breaking:** Emoji API: changed methods and renamed routes.
  </details>
  
 +<details>
 +  <summary>Admin API Changes</summary>
 +
 +- Status visibility stats: now can return stats per instance.
 +
 +- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
 +</details>
 +
  ### Removed
  - **Breaking:** removed `with_move` parameter from notifications timeline.
  
@@@ -53,6 -44,7 +53,7 @@@
  
  ### Fixed
  - Healthcheck reporting the number of memory currently used, rather than allocated in total
+ - `InsertSkeletonsForDeletedUsers` failing on some instances 
  
  ## [2.0.3] - 2020-05-02
  
@@@ -85,9 -77,6 +86,9 @@@
  2. Run database migrations (inside Pleroma directory):
    - OTP: `./bin/pleroma_ctl migrate`
    - From Source: `mix ecto.migrate`
 +3. Reset status visibility counters (inside Pleroma directory):
 +  - OTP: `./bin/pleroma_ctl refresh_counter_cache`
 +  - From Source: `mix pleroma.refresh_counter_cache`
  
  
  ## [2.0.2] - 2020-04-08

diff --combined lib/pleroma/web/admin_api/admin_api_controller.ex
index 4db9f4cac24394cae07e9846d3dcd03147ed69a1,9821173d0569a25314fb8456c0ffad721490da2c..9175b190442eb173ab7c71f777b20aa9edf9b072
--- 1/lib/pleroma/web/admin_api/admin_api_controller.ex
--- 2/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@@ -844,15 -844,20 +844,20 @@@ defmodule Pleroma.Web.AdminAPI.AdminAPI
    end
  
    def status_update(%{assigns: %{user: admin}} = conn, %{"id" => id} = params) do
+     params =
+       params
+       |> Map.take(["sensitive", "visibility"])
+       |> Map.new(fn {key, value} -> {String.to_existing_atom(key), value} end)
+ 
      with {:ok, activity} <- CommonAPI.update_activity_scope(id, params) do
-       {:ok, sensitive} = Ecto.Type.cast(:boolean, params["sensitive"])
+       {:ok, sensitive} = Ecto.Type.cast(:boolean, params[:sensitive])
  
        ModerationLog.insert_log(%{
          action: "status_update",
          actor: admin,
          subject: activity,
          sensitive: sensitive,
-         visibility: params["visibility"]
+         visibility: params[:visibility]
        })
  
        conn
@@@ -1122,10 -1127,11 +1127,10 @@@
      end
    end
  
 -  def stats(conn, _) do
 -    count = Stats.get_status_visibility_count()
 +  def stats(conn, params) do
 +    counters = Stats.get_status_visibility_count(params["instance"])
  
 -    conn
 -    |> json(%{"status_visibility" => count})
 +    json(conn, %{"status_visibility" => counters})
    end
  
    defp errors(conn, {:error, :not_found}) do

diff --combined test/stats_test.exs
index 33ed0b7dd12ab4492b75bf7132bf76e5dd8d69b4,4b76e2e782b12de65e9d9f94e343b58e28fc7239..f09d8d31a689f69fc52a211cc538695fbc599ede
--- 1/test/stats_test.exs
--- 2/test/stats_test.exs
+++ b/test/stats_test.exs
@@@ -17,103 -17,64 +17,103 @@@ defmodule Pleroma.StatsTest d
      end
    end
  
 -  describe "status visibility count" do
 +  describe "status visibility sum count" do
      test "on new status" do
 +      instance2 = "instance2.tld"
        user = insert(:user)
 -      other_user = insert(:user)
 +      other_user = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
  
-       CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+       CommonAPI.post(user, %{visibility: "public", status: "hey"})
  
        Enum.each(0..1, fn _ ->
          CommonAPI.post(user, %{
-           "visibility" => "unlisted",
-           "status" => "hey"
+           visibility: "unlisted",
+           status: "hey"
          })
        end)
  
        Enum.each(0..2, fn _ ->
          CommonAPI.post(user, %{
-           "visibility" => "direct",
-           "status" => "hey @#{other_user.nickname}"
+           visibility: "direct",
+           status: "hey @#{other_user.nickname}"
          })
        end)
  
        Enum.each(0..3, fn _ ->
          CommonAPI.post(user, %{
-           "visibility" => "private",
-           "status" => "hey"
+           visibility: "private",
+           status: "hey"
          })
        end)
  
 -      assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
 +      assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
                 Pleroma.Stats.get_status_visibility_count()
      end
  
      test "on status delete" do
        user = insert(:user)
-       {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+       {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
 -      assert %{public: 1} = Pleroma.Stats.get_status_visibility_count()
 +      assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count()
        CommonAPI.delete(activity.id, user)
 -      assert %{public: 0} = Pleroma.Stats.get_status_visibility_count()
 +      assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count()
      end
  
      test "on status visibility update" do
        user = insert(:user)
-       {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+       {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
 -      assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count()
 +      assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count()
-       {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{"visibility" => "private"})
+       {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
 -      assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count()
 +      assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count()
      end
  
      test "doesn't count unrelated activities" do
        user = insert(:user)
        other_user = insert(:user)
-       {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+       {:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
        _ = CommonAPI.follow(user, other_user)
        CommonAPI.favorite(other_user, activity.id)
        CommonAPI.repeat(activity.id, other_user)
  
 -      assert %{direct: 0, private: 0, public: 1, unlisted: 0} =
 +      assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
                 Pleroma.Stats.get_status_visibility_count()
      end
    end
-       CommonAPI.post(user1, %{"visibility" => "public", "status" => "hey"})
 +
 +  describe "status visibility by instance count" do
 +    test "single instance" do
 +      local_instance = Pleroma.Web.Endpoint.url() |> String.split("//") |> Enum.at(1)
 +      instance2 = "instance2.tld"
 +      user1 = insert(:user)
 +      user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
 +
-           "visibility" => "unlisted",
-           "status" => "hey"
++      CommonAPI.post(user1, %{visibility: "public", status: "hey"})
 +
 +      Enum.each(1..5, fn _ ->
 +        CommonAPI.post(user1, %{
-           "visibility" => "direct",
-           "status" => "hey @#{user2.nickname}"
++          visibility: "unlisted",
++          status: "hey"
 +        })
 +      end)
 +
 +      Enum.each(1..10, fn _ ->
 +        CommonAPI.post(user1, %{
-           "visibility" => "private",
-           "status" => "hey"
++          visibility: "direct",
++          status: "hey @#{user2.nickname}"
 +        })
 +      end)
 +
 +      Enum.each(1..20, fn _ ->
 +        CommonAPI.post(user2, %{
++          visibility: "private",
++          status: "hey"
 +        })
 +      end)
 +
 +      assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
 +               Pleroma.Stats.get_status_visibility_count(local_instance)
 +
 +      assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
 +               Pleroma.Stats.get_status_visibility_count(instance2)
 +    end
 +  end
  end

diff --combined test/tasks/refresh_counter_cache_test.exs
index 378664148c2cadd99bb74f2aad9b758316f959df,851971a778ef42cbe1c6e5f24b2094a70d11668d..6a1a9ac1741a3d804afc69efd535ebfa071f00d7
--- 1/test/tasks/refresh_counter_cache_test.exs
--- 2/test/tasks/refresh_counter_cache_test.exs
+++ b/test/tasks/refresh_counter_cache_test.exs
@@@ -12,32 -12,32 +12,32 @@@ defmodule Mix.Tasks.Pleroma.RefreshCoun
      user = insert(:user)
      other_user = insert(:user)
  
-     CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+     CommonAPI.post(user, %{visibility: "public", status: "hey"})
  
      Enum.each(0..1, fn _ ->
        CommonAPI.post(user, %{
-         "visibility" => "unlisted",
-         "status" => "hey"
+         visibility: "unlisted",
+         status: "hey"
        })
      end)
  
      Enum.each(0..2, fn _ ->
        CommonAPI.post(user, %{
-         "visibility" => "direct",
-         "status" => "hey @#{other_user.nickname}"
+         visibility: "direct",
+         status: "hey @#{other_user.nickname}"
        })
      end)
  
      Enum.each(0..3, fn _ ->
        CommonAPI.post(user, %{
-         "visibility" => "private",
-         "status" => "hey"
+         visibility: "private",
+         status: "hey"
        })
      end)
  
      assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
  
 -    assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
 +    assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
               Pleroma.Stats.get_status_visibility_count()
    end
  end

diff --combined test/web/admin_api/admin_api_controller_test.exs
index c3de89ac0d0d0f33fcafb4b2be5dd27cc8853d4a,e4b0dd627347ae52e8b3ef00ebeff706b387d6b3..dd0c6b88d19a70e3d8179ef67a0cb0cde79c4b5e
--- 1/test/web/admin_api/admin_api_controller_test.exs
--- 2/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@@@ -1747,7 -1747,7 +1747,7 @@@ defmodule Pleroma.Web.AdminAPI.AdminAPI
      test "change visibility flag", %{conn: conn, id: id, admin: admin} do
        response =
          conn
-         |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "public"})
+         |> put("/api/pleroma/admin/statuses/#{id}", %{visibility: "public"})
          |> json_response(:ok)
  
        assert response["visibility"] == "public"
@@@ -1759,21 -1759,21 +1759,21 @@@
  
        response =
          conn
-         |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "private"})
+         |> put("/api/pleroma/admin/statuses/#{id}", %{visibility: "private"})
          |> json_response(:ok)
  
        assert response["visibility"] == "private"
  
        response =
          conn
-         |> put("/api/pleroma/admin/statuses/#{id}", %{"visibility" => "unlisted"})
+         |> put("/api/pleroma/admin/statuses/#{id}", %{visibility: "unlisted"})
          |> json_response(:ok)
  
        assert response["visibility"] == "unlisted"
      end
  
      test "returns 400 when visibility is unknown", %{conn: conn, id: id} do
-       conn = put(conn, "/api/pleroma/admin/statuses/#{id}", %{"visibility" => "test"})
+       conn = put(conn, "/api/pleroma/admin/statuses/#{id}", %{visibility: "test"})
  
        assert json_response(conn, :bad_request) == "Unsupported visibility"
      end
@@@ -2862,26 -2862,25 +2862,25 @@@
                group: ":pleroma",
                key: ":http",
                value: [
-                 %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]},
-                 %{"tuple" => [":send_user_agent", false]}
+                 %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]}
                ]
              }
            ]
          })
  
-       assert json_response(conn, 200) == %{
+       assert %{
                 "configs" => [
                   %{
                     "group" => ":pleroma",
                     "key" => ":http",
-                    "value" => [
-                      %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]},
-                      %{"tuple" => [":send_user_agent", false]}
-                    ],
-                    "db" => [":proxy_url", ":send_user_agent"]
+                    "value" => value,
+                    "db" => db
                   }
                 ]
-              }
+              } = json_response(conn, 200)
+ 
+       assert %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "localhost", 1234]}]} in value
+       assert ":proxy_url" in db
      end
  
      test "proxy tuple domain", %{conn: conn} do
@@@ -2892,26 -2891,25 +2891,25 @@@
                group: ":pleroma",
                key: ":http",
                value: [
-                 %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]},
-                 %{"tuple" => [":send_user_agent", false]}
+                 %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]}
                ]
              }
            ]
          })
  
-       assert json_response(conn, 200) == %{
+       assert %{
                 "configs" => [
                   %{
                     "group" => ":pleroma",
                     "key" => ":http",
-                    "value" => [
-                      %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]},
-                      %{"tuple" => [":send_user_agent", false]}
-                    ],
-                    "db" => [":proxy_url", ":send_user_agent"]
+                    "value" => value,
+                    "db" => db
                   }
                 ]
-              }
+              } = json_response(conn, 200)
+ 
+       assert %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "domain.com", 1234]}]} in value
+       assert ":proxy_url" in db
      end
  
      test "proxy tuple ip", %{conn: conn} do
@@@ -2922,26 -2920,25 +2920,25 @@@
                group: ":pleroma",
                key: ":http",
                value: [
-                 %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]},
-                 %{"tuple" => [":send_user_agent", false]}
+                 %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]}
                ]
              }
            ]
          })
  
-       assert json_response(conn, 200) == %{
+       assert %{
                 "configs" => [
                   %{
                     "group" => ":pleroma",
                     "key" => ":http",
-                    "value" => [
-                      %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]},
-                      %{"tuple" => [":send_user_agent", false]}
-                    ],
-                    "db" => [":proxy_url", ":send_user_agent"]
+                    "value" => value,
+                    "db" => db
                   }
                 ]
-              }
+              } = json_response(conn, 200)
+ 
+       assert %{"tuple" => [":proxy_url", %{"tuple" => [":socks5", "127.0.0.1", 1234]}]} in value
+       assert ":proxy_url" in db
      end
    end
  
@@@ -2977,13 -2974,12 +2974,12 @@@
        user = insert(:user)
        User.block(admin, blocked)
  
-       {:ok, _} =
-         CommonAPI.post(user, %{"status" => "@#{admin.nickname}", "visibility" => "direct"})
+       {:ok, _} = CommonAPI.post(user, %{status: "@#{admin.nickname}", visibility: "direct"})
  
-       {:ok, _} = CommonAPI.post(user, %{"status" => ".", "visibility" => "unlisted"})
-       {:ok, _} = CommonAPI.post(user, %{"status" => ".", "visibility" => "private"})
-       {:ok, _} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"})
-       {:ok, _} = CommonAPI.post(blocked, %{"status" => ".", "visibility" => "public"})
+       {:ok, _} = CommonAPI.post(user, %{status: ".", visibility: "unlisted"})
+       {:ok, _} = CommonAPI.post(user, %{status: ".", visibility: "private"})
+       {:ok, _} = CommonAPI.post(user, %{status: ".", visibility: "public"})
+       {:ok, _} = CommonAPI.post(blocked, %{status: ".", visibility: "public"})
  
        response =
          conn
@@@ -3011,11 -3007,10 +3007,10 @@@
      test "returns private and direct statuses with godmode on", %{conn: conn, admin: admin} do
        user = insert(:user)
  
-       {:ok, _} =
-         CommonAPI.post(user, %{"status" => "@#{admin.nickname}", "visibility" => "direct"})
+       {:ok, _} = CommonAPI.post(user, %{status: "@#{admin.nickname}", visibility: "direct"})
  
-       {:ok, _} = CommonAPI.post(user, %{"status" => ".", "visibility" => "private"})
-       {:ok, _} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"})
+       {:ok, _} = CommonAPI.post(user, %{status: ".", visibility: "private"})
+       {:ok, _} = CommonAPI.post(user, %{status: ".", visibility: "public"})
        conn = get(conn, "/api/pleroma/admin/statuses?godmode=true")
        assert json_response(conn, 200) |> length() == 3
      end
@@@ -3049,11 -3044,9 +3044,9 @@@
      end
  
      test "doesn't return private statuses by default", %{conn: conn, user: user} do
-       {:ok, _private_status} =
-         CommonAPI.post(user, %{"status" => "private", "visibility" => "private"})
+       {:ok, _private_status} = CommonAPI.post(user, %{status: "private", visibility: "private"})
  
-       {:ok, _public_status} =
-         CommonAPI.post(user, %{"status" => "public", "visibility" => "public"})
+       {:ok, _public_status} = CommonAPI.post(user, %{status: "public", visibility: "public"})
  
        conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}/statuses")
  
@@@ -3061,11 -3054,9 +3054,9 @@@
      end
  
      test "returns private statuses with godmode on", %{conn: conn, user: user} do
-       {:ok, _private_status} =
-         CommonAPI.post(user, %{"status" => "private", "visibility" => "private"})
+       {:ok, _private_status} = CommonAPI.post(user, %{status: "private", visibility: "private"})
  
-       {:ok, _public_status} =
-         CommonAPI.post(user, %{"status" => "public", "visibility" => "public"})
+       {:ok, _public_status} = CommonAPI.post(user, %{status: "public", visibility: "public"})
  
        conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}/statuses?godmode=true")
  
@@@ -3074,7 -3065,7 +3065,7 @@@
  
      test "excludes reblogs by default", %{conn: conn, user: user} do
        other_user = insert(:user)
-       {:ok, activity} = CommonAPI.post(user, %{"status" => "."})
+       {:ok, activity} = CommonAPI.post(user, %{status: "."})
        {:ok, %Activity{}, _} = CommonAPI.repeat(activity.id, other_user)
  
        conn_res = get(conn, "/api/pleroma/admin/users/#{other_user.nickname}/statuses")
@@@ -3599,9 -3590,9 +3590,9 @@@
      test "status visibility count", %{conn: conn} do
        admin = insert(:user, is_admin: true)
        user = insert(:user)
-       CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
-       CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
-       CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
+       CommonAPI.post(user, %{visibility: "public", status: "hey"})
+       CommonAPI.post(user, %{visibility: "unlisted", status: "hey"})
+       CommonAPI.post(user, %{visibility: "unlisted", status: "hey"})
  
        response =
          conn
@@@ -3612,26 -3603,6 +3603,26 @@@
        assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
                 response["status_visibility"]
      end
-       CommonAPI.post(user1, %{"visibility" => "public", "status" => "hey"})
-       CommonAPI.post(user2, %{"visibility" => "unlisted", "status" => "hey"})
-       CommonAPI.post(user2, %{"visibility" => "private", "status" => "hey"})
 +
 +    test "by instance", %{conn: conn} do
 +      admin = insert(:user, is_admin: true)
 +      user1 = insert(:user)
 +      instance2 = "instance2.tld"
 +      user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
 +
++      CommonAPI.post(user1, %{visibility: "public", status: "hey"})
++      CommonAPI.post(user2, %{visibility: "unlisted", status: "hey"})
++      CommonAPI.post(user2, %{visibility: "private", status: "hey"})
 +
 +      response =
 +        conn
 +        |> assign(:user, admin)
 +        |> get("/api/pleroma/admin/stats", instance: instance2)
 +        |> json_response(200)
 +
 +      assert %{"direct" => 0, "private" => 1, "public" => 0, "unlisted" => 1} =
 +               response["status_visibility"]
 +    end
    end
  
    describe "POST /api/pleroma/admin/oauth_app" do