git.squeep.com Git - akkoma/blob - lib/pleroma/instances/instance.ex

   1 # Pleroma: A lightweight social networking server
   2 # Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
   3 # SPDX-License-Identifier: AGPL-3.0-only
   4
   5 defmodule Pleroma.Instances.Instance do
   6   @moduledoc "Instance."
   7
   8   @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
   9
  10   alias Pleroma.Instances
  11   alias Pleroma.Instances.Instance
  12   alias Pleroma.Repo
  13   alias Pleroma.User
  14   alias Pleroma.Workers.BackgroundWorker
  15
  16   use Ecto.Schema
  17
  18   import Ecto.Query
  19   import Ecto.Changeset
  20
  21   require Logger
  22
  23   schema "instances" do
  24     field(:host, :string)
  25     field(:unreachable_since, :naive_datetime_usec)
  26     field(:favicon, :string)
  27     field(:metadata_updated_at, :naive_datetime)
  28     field(:nodeinfo, :map, default: %{})
  29     field(:has_request_signatures, :boolean)
  30
  31     timestamps()
  32   end
  33
  34   defdelegate host(url_or_host), to: Instances
  35
  36   def changeset(struct, params \\ %{}) do
  37     struct
  38     |> cast(params, [
  39       :host,
  40       :unreachable_since,
  41       :favicon,
  42       :nodeinfo,
  43       :metadata_updated_at,
  44       :has_request_signatures
  45     ])
  46     |> validate_required([:host])
  47     |> unique_constraint(:host)
  48   end
  49
  50   def filter_reachable([]), do: %{}
  51
  52   def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
  53     hosts =
  54       urls_or_hosts
  55       |> Enum.map(&(&1 && host(&1)))
  56       |> Enum.filter(&(to_string(&1) != ""))
  57
  58     unreachable_since_by_host =
  59       Repo.all(
  60         from(i in Instance,
  61           where: i.host in ^hosts,
  62           select: {i.host, i.unreachable_since}
  63         )
  64       )
  65       |> Map.new(& &1)
  66
  67     reachability_datetime_threshold = Instances.reachability_datetime_threshold()
  68
  69     for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
  70       host = host(entry)
  71       unreachable_since = unreachable_since_by_host[host]
  72
  73       if !unreachable_since ||
  74            NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
  75         {entry, unreachable_since}
  76       end
  77     end
  78     |> Enum.filter(& &1)
  79     |> Map.new(& &1)
  80   end
  81
  82   def reachable?(url_or_host) when is_binary(url_or_host) do
  83     !Repo.one(
  84       from(i in Instance,
  85         where:
  86           i.host == ^host(url_or_host) and
  87             i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
  88         select: true
  89       )
  90     )
  91   end
  92
  93   def reachable?(url_or_host) when is_binary(url_or_host), do: true
  94
  95   def set_reachable(url_or_host) when is_binary(url_or_host) do
  96     with host <- host(url_or_host),
  97          %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
  98       {:ok, _instance} =
  99         existing_record
 100         |> changeset(%{unreachable_since: nil})
 101         |> Repo.update()
 102     end
 103   end
 104
 105   def set_reachable(_), do: {:error, nil}
 106
 107   def set_unreachable(url_or_host, unreachable_since \\ nil)
 108
 109   def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
 110     unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now()
 111     host = host(url_or_host)
 112     existing_record = Repo.get_by(Instance, %{host: host})
 113
 114     changes = %{unreachable_since: unreachable_since}
 115
 116     cond do
 117       is_nil(existing_record) ->
 118         %Instance{}
 119         |> changeset(Map.put(changes, :host, host))
 120         |> Repo.insert()
 121
 122       existing_record.unreachable_since &&
 123           NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
 124         {:ok, existing_record}
 125
 126       true ->
 127         existing_record
 128         |> changeset(changes)
 129         |> Repo.update()
 130     end
 131   end
 132
 133   def set_unreachable(_, _), do: {:error, nil}
 134
 135   def get_consistently_unreachable do
 136     reachability_datetime_threshold = Instances.reachability_datetime_threshold()
 137
 138     from(i in Instance,
 139       where: ^reachability_datetime_threshold > i.unreachable_since,
 140       order_by: i.unreachable_since,
 141       select: {i.host, i.unreachable_since}
 142     )
 143     |> Repo.all()
 144   end
 145
 146   defp parse_datetime(datetime) when is_binary(datetime) do
 147     NaiveDateTime.from_iso8601(datetime)
 148   end
 149
 150   defp parse_datetime(datetime), do: datetime
 151
 152   def needs_update(nil), do: true
 153
 154   def needs_update(%Instance{metadata_updated_at: nil}), do: true
 155
 156   def needs_update(%Instance{metadata_updated_at: metadata_updated_at}) do
 157     now = NaiveDateTime.utc_now()
 158     NaiveDateTime.diff(now, metadata_updated_at) > 86_400
 159   end
 160
 161   def local do
 162     %Instance{
 163       host: Pleroma.Web.Endpoint.host(),
 164       favicon: Pleroma.Web.Endpoint.url() <> "/favicon.png",
 165       nodeinfo: Pleroma.Web.Nodeinfo.Nodeinfo.get_nodeinfo("2.1")
 166     }
 167   end
 168
 169   def update_metadata(%URI{host: host} = uri) do
 170     Logger.debug("Checking metadata for #{host}")
 171     existing_record = Repo.get_by(Instance, %{host: host})
 172
 173     if reachable?(host) do
 174       do_update_metadata(uri, existing_record)
 175     else
 176       {:discard, :unreachable}
 177     end
 178   end
 179
 180   defp do_update_metadata(%URI{host: host} = uri, existing_record) do
 181     if existing_record do
 182       if needs_update(existing_record) do
 183         Logger.info("Updating metadata for #{host}")
 184         favicon = scrape_favicon(uri)
 185         nodeinfo = scrape_nodeinfo(uri)
 186
 187         existing_record
 188         |> changeset(%{
 189           host: host,
 190           favicon: favicon,
 191           nodeinfo: nodeinfo,
 192           metadata_updated_at: NaiveDateTime.utc_now()
 193         })
 194         |> Repo.update()
 195       else
 196         {:discard, "Does not require update"}
 197       end
 198     else
 199       favicon = scrape_favicon(uri)
 200       nodeinfo = scrape_nodeinfo(uri)
 201
 202       Logger.info("Creating metadata for #{host}")
 203
 204       %Instance{}
 205       |> changeset(%{
 206         host: host,
 207         favicon: favicon,
 208         nodeinfo: nodeinfo,
 209         metadata_updated_at: NaiveDateTime.utc_now()
 210       })
 211       |> Repo.insert()
 212     end
 213   end
 214
 215   def get_favicon(%URI{host: host}) do
 216     existing_record = Repo.get_by(Instance, %{host: host})
 217
 218     if existing_record do
 219       existing_record.favicon
 220     else
 221       nil
 222     end
 223   end
 224
 225   defp scrape_nodeinfo(%URI{} = instance_uri) do
 226     with true <- Pleroma.Config.get([:instances_nodeinfo, :enabled]),
 227          {_, true} <- {:reachable, reachable?(instance_uri.host)},
 228          {:ok, %Tesla.Env{status: 200, body: body}} <-
 229            Tesla.get(
 230              "https://#{instance_uri.host}/.well-known/nodeinfo",
 231              headers: [{"Accept", "application/json"}]
 232            ),
 233          {:ok, json} <- Jason.decode(body),
 234          {:ok, %{"links" => links}} <- {:ok, json},
 235          {:ok, %{"href" => href}} <-
 236            {:ok,
 237             Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))},
 238          {:ok, %Tesla.Env{body: data}} <-
 239            Pleroma.HTTP.get(href, [{"accept", "application/json"}], []),
 240          {:length, true} <- {:length, String.length(data) < 50_000},
 241          {:ok, nodeinfo} <- Jason.decode(data) do
 242       nodeinfo
 243     else
 244       {:reachable, false} ->
 245         Logger.debug(
 246           "Instance.scrape_nodeinfo(\"#{to_string(instance_uri)}\") ignored unreachable host"
 247         )
 248
 249         nil
 250
 251       {:length, false} ->
 252         Logger.debug(
 253           "Instance.scrape_nodeinfo(\"#{to_string(instance_uri)}\") ignored too long body"
 254         )
 255
 256         nil
 257
 258       _ ->
 259         nil
 260     end
 261   end
 262
 263   defp scrape_favicon(%URI{} = instance_uri) do
 264     with true <- Pleroma.Config.get([:instances_favicons, :enabled]),
 265          {_, true} <- {:reachable, reachable?(instance_uri.host)},
 266          {:ok, %Tesla.Env{body: html}} <-
 267            Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []),
 268          {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
 269            {:parse, html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
 270          {_, favicon} when is_binary(favicon) <-
 271            {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()},
 272          {:length, true} <- {:length, String.length(favicon) < 255} do
 273       favicon
 274     else
 275       {:reachable, false} ->
 276         Logger.debug(
 277           "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host"
 278         )
 279
 280         nil
 281
 282       _ ->
 283         nil
 284     end
 285   end
 286
 287   @doc """
 288   Deletes all users from an instance in a background task, thus also deleting
 289   all of those users' activities and notifications.
 290   """
 291   def delete_users_and_activities(host) when is_binary(host) do
 292     BackgroundWorker.enqueue("delete_instance", %{"host" => host})
 293   end
 294
 295   def perform(:delete_instance, host) when is_binary(host) do
 296     User.Query.build(%{nickname: "@#{host}"})
 297     |> Repo.chunk_stream(100, :batches)
 298     |> Stream.each(fn users ->
 299       users
 300       |> Enum.each(fn user ->
 301         User.perform(:delete, user)
 302       end)
 303     end)
 304     |> Stream.run()
 305   end
 306
 307   def get_by_url(url_or_host) do
 308     url = host(url_or_host)
 309     Repo.get_by(Instance, host: url)
 310   end
 311
 312   def get_cached_by_url(url_or_host) do
 313     url = host(url_or_host)
 314
 315     if url == Pleroma.Web.Endpoint.host() do
 316       {:ok, local()}
 317     else
 318       @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ ->
 319         with %Instance{} = instance <- get_by_url(url) do
 320           {:commit, {:ok, instance}}
 321         else
 322           _ -> {:ignore, nil}
 323         end
 324       end)
 325     end
 326   end
 327
 328   def set_request_signatures(url_or_host) when is_binary(url_or_host) do
 329     host = host(url_or_host)
 330     existing_record = Repo.get_by(Instance, %{host: host})
 331     changes = %{has_request_signatures: true}
 332
 333     cond do
 334       is_nil(existing_record) ->
 335         %Instance{}
 336         |> changeset(Map.put(changes, :host, host))
 337         |> Repo.insert()
 338
 339       true ->
 340         existing_record
 341         |> changeset(changes)
 342         |> Repo.update()
 343     end
 344   end
 345
 346   def set_request_signatures(_), do: {:error, :invalid_input}
 347 end