X-Git-Url: https://git.squeep.com/?a=blobdiff_plain;f=lib%2Fpleroma%2Finstances%2Finstance.ex;h=4d0e8034d61400f623169a0d47c3bc21f236e1a6;hb=fc42e714e234542bfec5a53a3fb9b7a45bcb4bd3;hp=fe52331a359c97eb620dc3d8cc106208c5314462;hpb=8654a591f08c7d8d5d61f075906f0c6907e877bb;p=akkoma diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index fe52331a3..4d0e8034d 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -1,85 +1,198 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Instances.Instance do @moduledoc "Instance." alias Pleroma.Instances alias Pleroma.Instances.Instance + alias Pleroma.Repo use Ecto.Schema - import Ecto.{Query, Changeset} + import Ecto.Query + import Ecto.Changeset - alias Pleroma.Repo + require Logger schema "instances" do field(:host, :string) - field(:unreachable_since, :naive_datetime) - field(:reachability_checked_at, :naive_datetime) + field(:unreachable_since, :naive_datetime_usec) + field(:favicon, :string) + field(:favicon_updated_at, :naive_datetime) timestamps() end - def update_changeset(struct, params \\ %{}) do + defdelegate host(url_or_host), to: Instances + + def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :reachability_checked_at]) + |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) + |> validate_required([:host]) |> unique_constraint(:host) end - def reachable?(url) when is_binary(url) do + def filter_reachable([]), do: %{} + + def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do + hosts = + urls_or_hosts + |> Enum.map(&(&1 && host(&1))) + |> Enum.filter(&(to_string(&1) != "")) + + unreachable_since_by_host = + Repo.all( + from(i in Instance, + where: i.host in ^hosts, + select: {i.host, i.unreachable_since} + ) + ) + |> Map.new(& &1) + + reachability_datetime_threshold = Instances.reachability_datetime_threshold() + + for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do + host = host(entry) + unreachable_since = unreachable_since_by_host[host] + + if !unreachable_since || + NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do + {entry, unreachable_since} + end + end + |> Enum.filter(& &1) + |> Map.new(& &1) + end + + def reachable?(url_or_host) when is_binary(url_or_host) do !Repo.one( from(i in Instance, where: - i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(), + i.host == ^host(url_or_host) and + i.unreachable_since <= ^Instances.reachability_datetime_threshold(), select: true ) ) end - def reachable?(_), do: true + def reachable?(url_or_host) when is_binary(url_or_host), do: true - def set_reachable(url) when is_binary(url) do - Repo.update_all( - from(i in Instance, where: i.host == ^host(url)), - set: [ - unreachable_since: nil, - reachability_checked_at: DateTime.utc_now() - ] - ) + def set_reachable(url_or_host) when is_binary(url_or_host) do + with host <- host(url_or_host), + %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do + {:ok, _instance} = + existing_record + |> changeset(%{unreachable_since: nil}) + |> Repo.update() + end end - def set_reachable(_), do: {0, :noop} + def set_reachable(_), do: {:error, nil} - def set_unreachable(url, unreachable_since \\ nil) + def set_unreachable(url_or_host, unreachable_since \\ nil) - def set_unreachable(url, unreachable_since) when is_binary(url) do - unreachable_since = unreachable_since || DateTime.utc_now() - host = host(url) + def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do + unreachable_since = parse_datetime(unreachable_since) || NaiveDateTime.utc_now() + host = host(url_or_host) existing_record = Repo.get_by(Instance, %{host: host}) - changes = %{ - unreachable_since: unreachable_since, - reachability_checked_at: NaiveDateTime.utc_now() - } + changes = %{unreachable_since: unreachable_since} - if existing_record do - update_changes = - if existing_record.unreachable_since && - NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt, - do: Map.delete(changes, :unreachable_since), - else: changes + cond do + is_nil(existing_record) -> + %Instance{} + |> changeset(Map.put(changes, :host, host)) + |> Repo.insert() - {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes)) - else - {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host))) + existing_record.unreachable_since && + NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt -> + {:ok, existing_record} + + true -> + existing_record + |> changeset(changes) + |> Repo.update() end end - def set_unreachable(_, _), do: {0, :noop} + def set_unreachable(_, _), do: {:error, nil} + + def get_consistently_unreachable do + reachability_datetime_threshold = Instances.reachability_datetime_threshold() + + from(i in Instance, + where: ^reachability_datetime_threshold > i.unreachable_since, + order_by: i.unreachable_since, + select: {i.host, i.unreachable_since} + ) + |> Repo.all() + end + + defp parse_datetime(datetime) when is_binary(datetime) do + NaiveDateTime.from_iso8601(datetime) + end + + defp parse_datetime(datetime), do: datetime + + def get_or_update_favicon(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + now = NaiveDateTime.utc_now() - defp host(url_or_host) do - if url_or_host =~ ~r/^http/i do - URI.parse(url_or_host).host + if existing_record && existing_record.favicon_updated_at && + NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do + existing_record.favicon else - url_or_host + favicon = scrape_favicon(instance_uri) + + if existing_record do + existing_record + |> changeset(%{favicon: favicon, favicon_updated_at: now}) + |> Repo.update() + else + %Instance{} + |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) + |> Repo.insert() + end + + favicon + end + rescue + e -> + Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") + nil + end + + defp scrape_favicon(%URI{} = instance_uri) do + try do + with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media), + {_, [favicon_rel | _]} when is_binary(favicon_rel) <- + {:parse, + html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, + {_, favicon} when is_binary(favicon) <- + {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do + favicon + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" + ) + + nil + + _ -> + nil + end + rescue + e -> + Logger.warn( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + ) + + nil end end end