git.squeep.com Git - akkoma/blob - test/pool/connections_test.exs

   1 # Pleroma: A lightweight social networking server
   2 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
   3 # SPDX-License-Identifier: AGPL-3.0-only
   4
   5 defmodule Pleroma.Pool.ConnectionsTest do
   6   use ExUnit.Case, async: true
   7   use Pleroma.Tests.Helpers
   8
   9   import ExUnit.CaptureLog
  10   import Mox
  11
  12   alias Pleroma.Gun.Conn
  13   alias Pleroma.GunMock
  14   alias Pleroma.Pool.Connections
  15
  16   setup :verify_on_exit!
  17
  18   setup_all do
  19     name = :test_connections
  20     {:ok, pid} = Connections.start_link({name, [checkin_timeout: 150]})
  21     {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock)
  22
  23     on_exit(fn ->
  24       if Process.alive?(pid), do: GenServer.stop(name)
  25     end)
  26
  27     {:ok, name: name}
  28   end
  29
  30   defp open_mock(num \\ 1) do
  31     GunMock
  32     |> expect(:open, num, &start_and_register(&1, &2, &3))
  33     |> expect(:await_up, num, fn _, _ -> {:ok, :http} end)
  34     |> expect(:set_owner, num, fn _, _ -> :ok end)
  35   end
  36
  37   defp connect_mock(mock) do
  38     mock
  39     |> expect(:connect, &connect(&1, &2))
  40     |> expect(:await, &await(&1, &2))
  41   end
  42
  43   defp info_mock(mock), do: expect(mock, :info, &info(&1))
  44
  45   defp start_and_register('gun-not-up.com', _, _), do: {:error, :timeout}
  46
  47   defp start_and_register(host, port, _) do
  48     {:ok, pid} = Task.start_link(fn -> Process.sleep(1000) end)
  49
  50     scheme =
  51       case port do
  52         443 -> "https"
  53         _ -> "http"
  54       end
  55
  56     Registry.register(GunMock, pid, %{
  57       origin_scheme: scheme,
  58       origin_host: host,
  59       origin_port: port
  60     })
  61
  62     {:ok, pid}
  63   end
  64
  65   defp info(pid) do
  66     [{_, info}] = Registry.lookup(GunMock, pid)
  67     info
  68   end
  69
  70   defp connect(pid, _) do
  71     ref = make_ref()
  72     Registry.register(GunMock, ref, pid)
  73     ref
  74   end
  75
  76   defp await(pid, ref) do
  77     [{_, ^pid}] = Registry.lookup(GunMock, ref)
  78     {:response, :fin, 200, []}
  79   end
  80
  81   defp now, do: :os.system_time(:second)
  82
  83   describe "alive?/2" do
  84     test "is alive", %{name: name} do
  85       assert Connections.alive?(name)
  86     end
  87
  88     test "returns false if not started" do
  89       refute Connections.alive?(:some_random_name)
  90     end
  91   end
  92
  93   test "opens connection and reuse it on next request", %{name: name} do
  94     open_mock()
  95     url = "http://some-domain.com"
  96     key = "http:some-domain.com:80"
  97     refute Connections.checkin(url, name)
  98     :ok = Conn.open(url, name)
  99
 100     conn = Connections.checkin(url, name)
 101     assert is_pid(conn)
 102     assert Process.alive?(conn)
 103
 104     self = self()
 105
 106     %Connections{
 107       conns: %{
 108         ^key => %Conn{
 109           conn: ^conn,
 110           gun_state: :up,
 111           used_by: [{^self, _}],
 112           conn_state: :active
 113         }
 114       }
 115     } = Connections.get_state(name)
 116
 117     reused_conn = Connections.checkin(url, name)
 118
 119     assert conn == reused_conn
 120
 121     %Connections{
 122       conns: %{
 123         ^key => %Conn{
 124           conn: ^conn,
 125           gun_state: :up,
 126           used_by: [{^self, _}, {^self, _}],
 127           conn_state: :active
 128         }
 129       }
 130     } = Connections.get_state(name)
 131
 132     :ok = Connections.checkout(conn, self, name)
 133
 134     %Connections{
 135       conns: %{
 136         ^key => %Conn{
 137           conn: ^conn,
 138           gun_state: :up,
 139           used_by: [{^self, _}],
 140           conn_state: :active
 141         }
 142       }
 143     } = Connections.get_state(name)
 144
 145     :ok = Connections.checkout(conn, self, name)
 146
 147     %Connections{
 148       conns: %{
 149         ^key => %Conn{
 150           conn: ^conn,
 151           gun_state: :up,
 152           used_by: [],
 153           conn_state: :idle
 154         }
 155       }
 156     } = Connections.get_state(name)
 157   end
 158
 159   test "reuse connection for idna domains", %{name: name} do
 160     open_mock()
 161     url = "http://ですsome-domain.com"
 162     refute Connections.checkin(url, name)
 163
 164     :ok = Conn.open(url, name)
 165
 166     conn = Connections.checkin(url, name)
 167     assert is_pid(conn)
 168     assert Process.alive?(conn)
 169
 170     self = self()
 171
 172     %Connections{
 173       conns: %{
 174         "http:ですsome-domain.com:80" => %Conn{
 175           conn: ^conn,
 176           gun_state: :up,
 177           used_by: [{^self, _}],
 178           conn_state: :active
 179         }
 180       }
 181     } = Connections.get_state(name)
 182
 183     reused_conn = Connections.checkin(url, name)
 184
 185     assert conn == reused_conn
 186   end
 187
 188   test "reuse for ipv4", %{name: name} do
 189     open_mock()
 190     url = "http://127.0.0.1"
 191
 192     refute Connections.checkin(url, name)
 193
 194     :ok = Conn.open(url, name)
 195
 196     conn = Connections.checkin(url, name)
 197     assert is_pid(conn)
 198     assert Process.alive?(conn)
 199
 200     self = self()
 201
 202     %Connections{
 203       conns: %{
 204         "http:127.0.0.1:80" => %Conn{
 205           conn: ^conn,
 206           gun_state: :up,
 207           used_by: [{^self, _}],
 208           conn_state: :active
 209         }
 210       }
 211     } = Connections.get_state(name)
 212
 213     reused_conn = Connections.checkin(url, name)
 214
 215     assert conn == reused_conn
 216
 217     :ok = Connections.checkout(conn, self, name)
 218     :ok = Connections.checkout(reused_conn, self, name)
 219
 220     %Connections{
 221       conns: %{
 222         "http:127.0.0.1:80" => %Conn{
 223           conn: ^conn,
 224           gun_state: :up,
 225           used_by: [],
 226           conn_state: :idle
 227         }
 228       }
 229     } = Connections.get_state(name)
 230   end
 231
 232   test "reuse for ipv6", %{name: name} do
 233     open_mock()
 234     url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
 235
 236     refute Connections.checkin(url, name)
 237
 238     :ok = Conn.open(url, name)
 239
 240     conn = Connections.checkin(url, name)
 241     assert is_pid(conn)
 242     assert Process.alive?(conn)
 243
 244     self = self()
 245
 246     %Connections{
 247       conns: %{
 248         "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
 249           conn: ^conn,
 250           gun_state: :up,
 251           used_by: [{^self, _}],
 252           conn_state: :active
 253         }
 254       }
 255     } = Connections.get_state(name)
 256
 257     reused_conn = Connections.checkin(url, name)
 258
 259     assert conn == reused_conn
 260   end
 261
 262   test "up and down ipv4", %{name: name} do
 263     open_mock()
 264     |> info_mock()
 265     |> allow(self(), name)
 266
 267     self = self()
 268     url = "http://127.0.0.1"
 269     :ok = Conn.open(url, name)
 270     conn = Connections.checkin(url, name)
 271     send(name, {:gun_down, conn, nil, nil, nil})
 272     send(name, {:gun_up, conn, nil})
 273
 274     %Connections{
 275       conns: %{
 276         "http:127.0.0.1:80" => %Conn{
 277           conn: ^conn,
 278           gun_state: :up,
 279           used_by: [{^self, _}],
 280           conn_state: :active
 281         }
 282       }
 283     } = Connections.get_state(name)
 284   end
 285
 286   test "up and down ipv6", %{name: name} do
 287     self = self()
 288
 289     open_mock()
 290     |> info_mock()
 291     |> allow(self, name)
 292
 293     url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
 294     :ok = Conn.open(url, name)
 295     conn = Connections.checkin(url, name)
 296     send(name, {:gun_down, conn, nil, nil, nil})
 297     send(name, {:gun_up, conn, nil})
 298
 299     %Connections{
 300       conns: %{
 301         "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
 302           conn: ^conn,
 303           gun_state: :up,
 304           used_by: [{^self, _}],
 305           conn_state: :active
 306         }
 307       }
 308     } = Connections.get_state(name)
 309   end
 310
 311   test "reuses connection based on protocol", %{name: name} do
 312     open_mock(2)
 313     http_url = "http://some-domain.com"
 314     http_key = "http:some-domain.com:80"
 315     https_url = "https://some-domain.com"
 316     https_key = "https:some-domain.com:443"
 317
 318     refute Connections.checkin(http_url, name)
 319     :ok = Conn.open(http_url, name)
 320     conn = Connections.checkin(http_url, name)
 321     assert is_pid(conn)
 322     assert Process.alive?(conn)
 323
 324     refute Connections.checkin(https_url, name)
 325     :ok = Conn.open(https_url, name)
 326     https_conn = Connections.checkin(https_url, name)
 327
 328     refute conn == https_conn
 329
 330     reused_https = Connections.checkin(https_url, name)
 331
 332     refute conn == reused_https
 333
 334     assert reused_https == https_conn
 335
 336     %Connections{
 337       conns: %{
 338         ^http_key => %Conn{
 339           conn: ^conn,
 340           gun_state: :up
 341         },
 342         ^https_key => %Conn{
 343           conn: ^https_conn,
 344           gun_state: :up
 345         }
 346       }
 347     } = Connections.get_state(name)
 348   end
 349
 350   test "connection can't get up", %{name: name} do
 351     expect(GunMock, :open, &start_and_register(&1, &2, &3))
 352     url = "http://gun-not-up.com"
 353
 354     assert capture_log(fn ->
 355              refute Conn.open(url, name)
 356              refute Connections.checkin(url, name)
 357            end) =~
 358              "Opening connection to http://gun-not-up.com failed with error {:error, :timeout}"
 359   end
 360
 361   test "process gun_down message and then gun_up", %{name: name} do
 362     self = self()
 363
 364     open_mock()
 365     |> info_mock()
 366     |> allow(self, name)
 367
 368     url = "http://gun-down-and-up.com"
 369     key = "http:gun-down-and-up.com:80"
 370     :ok = Conn.open(url, name)
 371     conn = Connections.checkin(url, name)
 372
 373     assert is_pid(conn)
 374     assert Process.alive?(conn)
 375
 376     %Connections{
 377       conns: %{
 378         ^key => %Conn{
 379           conn: ^conn,
 380           gun_state: :up,
 381           used_by: [{^self, _}]
 382         }
 383       }
 384     } = Connections.get_state(name)
 385
 386     send(name, {:gun_down, conn, :http, nil, nil})
 387
 388     %Connections{
 389       conns: %{
 390         ^key => %Conn{
 391           conn: ^conn,
 392           gun_state: :down,
 393           used_by: [{^self, _}]
 394         }
 395       }
 396     } = Connections.get_state(name)
 397
 398     send(name, {:gun_up, conn, :http})
 399
 400     conn2 = Connections.checkin(url, name)
 401     assert conn == conn2
 402
 403     assert is_pid(conn2)
 404     assert Process.alive?(conn2)
 405
 406     %Connections{
 407       conns: %{
 408         ^key => %Conn{
 409           conn: _,
 410           gun_state: :up,
 411           used_by: [{^self, _}, {^self, _}]
 412         }
 413       }
 414     } = Connections.get_state(name)
 415   end
 416
 417   test "async processes get same conn for same domain", %{name: name} do
 418     open_mock()
 419     url = "http://some-domain.com"
 420     :ok = Conn.open(url, name)
 421
 422     tasks =
 423       for _ <- 1..5 do
 424         Task.async(fn ->
 425           Connections.checkin(url, name)
 426         end)
 427       end
 428
 429     tasks_with_results = Task.yield_many(tasks)
 430
 431     results =
 432       Enum.map(tasks_with_results, fn {task, res} ->
 433         res || Task.shutdown(task, :brutal_kill)
 434       end)
 435
 436     conns = for {:ok, value} <- results, do: value
 437
 438     %Connections{
 439       conns: %{
 440         "http:some-domain.com:80" => %Conn{
 441           conn: conn,
 442           gun_state: :up
 443         }
 444       }
 445     } = Connections.get_state(name)
 446
 447     assert Enum.all?(conns, fn res -> res == conn end)
 448   end
 449
 450   test "remove frequently used and idle", %{name: name} do
 451     open_mock(3)
 452     self = self()
 453     http_url = "http://some-domain.com"
 454     https_url = "https://some-domain.com"
 455     :ok = Conn.open(https_url, name)
 456     :ok = Conn.open(http_url, name)
 457
 458     conn1 = Connections.checkin(https_url, name)
 459
 460     [conn2 | _conns] =
 461       for _ <- 1..4 do
 462         Connections.checkin(http_url, name)
 463       end
 464
 465     http_key = "http:some-domain.com:80"
 466
 467     %Connections{
 468       conns: %{
 469         ^http_key => %Conn{
 470           conn: ^conn2,
 471           gun_state: :up,
 472           conn_state: :active,
 473           used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}]
 474         },
 475         "https:some-domain.com:443" => %Conn{
 476           conn: ^conn1,
 477           gun_state: :up,
 478           conn_state: :active,
 479           used_by: [{^self, _}]
 480         }
 481       }
 482     } = Connections.get_state(name)
 483
 484     :ok = Connections.checkout(conn1, self, name)
 485
 486     another_url = "http://another-domain.com"
 487     :ok = Conn.open(another_url, name)
 488     conn = Connections.checkin(another_url, name)
 489
 490     %Connections{
 491       conns: %{
 492         "http:another-domain.com:80" => %Conn{
 493           conn: ^conn,
 494           gun_state: :up
 495         },
 496         ^http_key => %Conn{
 497           conn: _,
 498           gun_state: :up
 499         }
 500       }
 501     } = Connections.get_state(name)
 502   end
 503
 504   describe "with proxy" do
 505     test "as ip", %{name: name} do
 506       open_mock()
 507       |> connect_mock()
 508
 509       url = "http://proxy-string.com"
 510       key = "http:proxy-string.com:80"
 511       :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
 512
 513       conn = Connections.checkin(url, name)
 514
 515       %Connections{
 516         conns: %{
 517           ^key => %Conn{
 518             conn: ^conn,
 519             gun_state: :up
 520           }
 521         }
 522       } = Connections.get_state(name)
 523
 524       reused_conn = Connections.checkin(url, name)
 525
 526       assert reused_conn == conn
 527     end
 528
 529     test "as host", %{name: name} do
 530       open_mock()
 531       |> connect_mock()
 532
 533       url = "http://proxy-tuple-atom.com"
 534       :ok = Conn.open(url, name, proxy: {'localhost', 9050})
 535       conn = Connections.checkin(url, name)
 536
 537       %Connections{
 538         conns: %{
 539           "http:proxy-tuple-atom.com:80" => %Conn{
 540             conn: ^conn,
 541             gun_state: :up
 542           }
 543         }
 544       } = Connections.get_state(name)
 545
 546       reused_conn = Connections.checkin(url, name)
 547
 548       assert reused_conn == conn
 549     end
 550
 551     test "as ip and ssl", %{name: name} do
 552       open_mock()
 553       |> connect_mock()
 554
 555       url = "https://proxy-string.com"
 556
 557       :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
 558       conn = Connections.checkin(url, name)
 559
 560       %Connections{
 561         conns: %{
 562           "https:proxy-string.com:443" => %Conn{
 563             conn: ^conn,
 564             gun_state: :up
 565           }
 566         }
 567       } = Connections.get_state(name)
 568
 569       reused_conn = Connections.checkin(url, name)
 570
 571       assert reused_conn == conn
 572     end
 573
 574     test "as host and ssl", %{name: name} do
 575       open_mock()
 576       |> connect_mock()
 577
 578       url = "https://proxy-tuple-atom.com"
 579       :ok = Conn.open(url, name, proxy: {'localhost', 9050})
 580       conn = Connections.checkin(url, name)
 581
 582       %Connections{
 583         conns: %{
 584           "https:proxy-tuple-atom.com:443" => %Conn{
 585             conn: ^conn,
 586             gun_state: :up
 587           }
 588         }
 589       } = Connections.get_state(name)
 590
 591       reused_conn = Connections.checkin(url, name)
 592
 593       assert reused_conn == conn
 594     end
 595
 596     test "with socks type", %{name: name} do
 597       open_mock()
 598
 599       url = "http://proxy-socks.com"
 600
 601       :ok = Conn.open(url, name, proxy: {:socks5, 'localhost', 1234})
 602
 603       conn = Connections.checkin(url, name)
 604
 605       %Connections{
 606         conns: %{
 607           "http:proxy-socks.com:80" => %Conn{
 608             conn: ^conn,
 609             gun_state: :up
 610           }
 611         }
 612       } = Connections.get_state(name)
 613
 614       reused_conn = Connections.checkin(url, name)
 615
 616       assert reused_conn == conn
 617     end
 618
 619     test "with socks4 type and ssl", %{name: name} do
 620       open_mock()
 621       url = "https://proxy-socks.com"
 622
 623       :ok = Conn.open(url, name, proxy: {:socks4, 'localhost', 1234})
 624
 625       conn = Connections.checkin(url, name)
 626
 627       %Connections{
 628         conns: %{
 629           "https:proxy-socks.com:443" => %Conn{
 630             conn: ^conn,
 631             gun_state: :up
 632           }
 633         }
 634       } = Connections.get_state(name)
 635
 636       reused_conn = Connections.checkin(url, name)
 637
 638       assert reused_conn == conn
 639     end
 640   end
 641
 642   describe "crf/3" do
 643     setup do
 644       crf = Connections.crf(1, 10, 1)
 645       {:ok, crf: crf}
 646     end
 647
 648     test "more used will have crf higher", %{crf: crf} do
 649       # used 3 times
 650       crf1 = Connections.crf(1, 10, crf)
 651       crf1 = Connections.crf(1, 10, crf1)
 652
 653       # used 2 times
 654       crf2 = Connections.crf(1, 10, crf)
 655
 656       assert crf1 > crf2
 657     end
 658
 659     test "recently used will have crf higher on equal references", %{crf: crf} do
 660       # used 3 sec ago
 661       crf1 = Connections.crf(3, 10, crf)
 662
 663       # used 4 sec ago
 664       crf2 = Connections.crf(4, 10, crf)
 665
 666       assert crf1 > crf2
 667     end
 668
 669     test "equal crf on equal reference and time", %{crf: crf} do
 670       # used 2 times
 671       crf1 = Connections.crf(1, 10, crf)
 672
 673       # used 2 times
 674       crf2 = Connections.crf(1, 10, crf)
 675
 676       assert crf1 == crf2
 677     end
 678
 679     test "recently used will have higher crf", %{crf: crf} do
 680       crf1 = Connections.crf(2, 10, crf)
 681       crf1 = Connections.crf(1, 10, crf1)
 682
 683       crf2 = Connections.crf(3, 10, crf)
 684       crf2 = Connections.crf(4, 10, crf2)
 685       assert crf1 > crf2
 686     end
 687   end
 688
 689   describe "get_unused_conns/1" do
 690     test "crf is equalent, sorting by reference", %{name: name} do
 691       Connections.add_conn(name, "1", %Conn{
 692         conn_state: :idle,
 693         last_reference: now() - 1
 694       })
 695
 696       Connections.add_conn(name, "2", %Conn{
 697         conn_state: :idle,
 698         last_reference: now()
 699       })
 700
 701       assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
 702     end
 703
 704     test "reference is equalent, sorting by crf", %{name: name} do
 705       Connections.add_conn(name, "1", %Conn{
 706         conn_state: :idle,
 707         crf: 1.999
 708       })
 709
 710       Connections.add_conn(name, "2", %Conn{
 711         conn_state: :idle,
 712         crf: 2
 713       })
 714
 715       assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
 716     end
 717
 718     test "higher crf and lower reference", %{name: name} do
 719       Connections.add_conn(name, "1", %Conn{
 720         conn_state: :idle,
 721         crf: 3,
 722         last_reference: now() - 1
 723       })
 724
 725       Connections.add_conn(name, "2", %Conn{
 726         conn_state: :idle,
 727         crf: 2,
 728         last_reference: now()
 729       })
 730
 731       assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name)
 732     end
 733
 734     test "lower crf and lower reference", %{name: name} do
 735       Connections.add_conn(name, "1", %Conn{
 736         conn_state: :idle,
 737         crf: 1.99,
 738         last_reference: now() - 1
 739       })
 740
 741       Connections.add_conn(name, "2", %Conn{
 742         conn_state: :idle,
 743         crf: 2,
 744         last_reference: now()
 745       })
 746
 747       assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
 748     end
 749   end
 750
 751   test "count/1" do
 752     name = :test_count
 753     {:ok, _} = Connections.start_link({name, [checkin_timeout: 150]})
 754     assert Connections.count(name) == 0
 755     Connections.add_conn(name, "1", %Conn{conn: self()})
 756     assert Connections.count(name) == 1
 757     Connections.remove_conn(name, "1")
 758     assert Connections.count(name) == 0
 759   end
 760 end