From f7e70910f579079d04c4cbeff750c8721a51fbe6 Mon Sep 17 00:00:00 2001 From: Justin Wind Date: Thu, 29 Jun 2023 17:12:10 -0700 Subject: [PATCH] use got instead of axios, some cleanup, problem with async context being lost for logging --- config/default.js | 1 + package-lock.json | 352 ++++++++++---------------------- package.json | 6 +- server.js | 4 +- src/common.js | 29 ++- src/communication.js | 300 +++++++++++++-------------- src/db/postgres/index.js | 2 + src/manager.js | 8 +- src/service.js | 11 +- test-e2e/fake-servers-client.js | 140 +++++++------ test-e2e/fake-servers.js | 13 +- test-e2e/test-many.js | 7 +- test-e2e/test-one.js | 9 +- test/src/common.js | 52 +++-- test/src/communication.js | 307 +++++++++------------------- 15 files changed, 512 insertions(+), 729 deletions(-) diff --git a/config/default.js b/config/default.js index 3f54d9a..2d3f34e 100644 --- a/config/default.js +++ b/config/default.js @@ -60,6 +60,7 @@ const defaultOptions = { }, communication: { + requestTimeoutMs: 120000, strictTopicHubLink: true, // If true, deletes topics which do not list us (dingus.selfBaseUrl) as a hub relation. retryBackoffSeconds: [60, 120, 360, 1440, 7200, 43200, 86400], // failed requests retry according to number of attempts claimTimeoutSeconds: 600, // how long until an in-progress task is deemed abandoned diff --git a/package-lock.json b/package-lock.json index 4b3a041..234a66a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,13 +14,13 @@ "@squeep/html-template-helper": "git+https://git.squeep.com/squeep-html-template-helper#v1.4.0", "@squeep/logger-json-console": "^2.0.1", "@squeep/web-linking": "^1.0.8", - "axios": "^1.4.0", "feedparser": "^2.2.10", + "got": "^13.0.0", "htmlparser2": "^9.0.0", "iconv": "^3.0.1" }, "devDependencies": { - "eslint": "^8.42.0", + "eslint": "^8.43.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^6.1.1", "eslint-plugin-security": "^1.7.1", @@ -30,7 +30,7 @@ "mocha-steps": "^1.3.0", "nyc": "^15.1.0", "pre-commit": "^1.2.2", - "sinon": "^15.1.0" + "sinon": "^15.2.0" }, "engines": { "node": ">=14" @@ -489,9 +489,9 @@ } }, "node_modules/@eslint/js": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz", - "integrity": "sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw==", + "version": "8.43.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.43.0.tgz", + "integrity": "sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -677,9 +677,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", "dev": true }, "node_modules/@jridgewell/trace-mapping": { @@ -692,12 +692,6 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, - "node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", - "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", - "dev": true - }, "node_modules/@mapbox/node-pre-gyp": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.10.tgz", @@ -783,9 +777,9 @@ } }, "node_modules/@sinonjs/fake-timers": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.2.0.tgz", - "integrity": "sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", "dev": true, "dependencies": { "@sinonjs/commons": "^3.0.0" @@ -928,9 +922,9 @@ "optional": true }, "node_modules/acorn": { - "version": "8.8.2", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", - "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.9.0.tgz", + "integrity": "sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -1053,9 +1047,9 @@ } }, "node_modules/aproba": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", - "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "optional": true }, "node_modules/archy": { @@ -1126,21 +1120,6 @@ "node": ">=10.0.0" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" - }, - "node_modules/axios": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", - "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", - "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -1250,9 +1229,9 @@ "dev": true }, "node_modules/browserslist": { - "version": "4.21.7", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.7.tgz", - "integrity": "sha512-BauCXrQ7I2ftSqd2mvKHGo85XR0u7Ru3C/Hxsy/0TkfCtjrmAbPdzLGasmoiBxplpDXlPvdjX9u7srIMfgasNA==", + "version": "4.21.9", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.9.tgz", + "integrity": "sha512-M0MFoZzbUrRU4KNfCrDLnvyE7gub+peetoTid3TBIqtunaDJyXlwhakT+/VkvSXcfIzFfK/nkCs4nmyTmxdNSg==", "dev": true, "funding": [ { @@ -1269,8 +1248,8 @@ } ], "dependencies": { - "caniuse-lite": "^1.0.30001489", - "electron-to-chromium": "^1.4.411", + "caniuse-lite": "^1.0.30001503", + "electron-to-chromium": "^1.4.431", "node-releases": "^2.0.12", "update-browserslist-db": "^1.0.11" }, @@ -1329,12 +1308,12 @@ } }, "node_modules/cacheable-request": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.10.tgz", - "integrity": "sha512-v6WB+Epm/qO4Hdlio/sfUn69r5Shgh39SsE9DSd4bIezP0mblOlObI+I0kUEM7J0JFc+I7pSeMeYaOYtX1N/VQ==", + "version": "10.2.11", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.11.tgz", + "integrity": "sha512-kn0t0oJnlFo1Nzl/AYQzS/oByMtmaqLasFUa7MUMsiTrIHy8TxSkx2KzWCybE3Nuz1F4sJRGnLAfUGsPe47viQ==", "dependencies": { "@types/http-cache-semantics": "^4.0.1", - "get-stream": "^6.0.1", + "get-stream": "^7.0.0", "http-cache-semantics": "^4.1.1", "keyv": "^4.5.2", "mimic-response": "^4.0.0", @@ -1345,6 +1324,17 @@ "node": ">=14.16" } }, + "node_modules/cacheable-request/node_modules/get-stream": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-7.0.0.tgz", + "integrity": "sha512-ql6FW5b8tgMYvI4UaoxG3EQN3VyZ6VeQpxNBGg5BZ4xD4u+HJeprzhMMA4OCBEGQgSR+m87pstWMpiVW64W8Fw==", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -1389,9 +1379,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001498", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001498.tgz", - "integrity": "sha512-LFInN2zAwx3ANrGCDZ5AKKJroHqNKyjXitdV5zRIVIaQlXKj3GmxUKagoKsjqUfckpAObPCEWnk5EeMlyMWcgw==", + "version": "1.0.30001505", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001505.tgz", + "integrity": "sha512-jaAOR5zVtxHfL0NjZyflVTtXm3D3J9P15zSJ7HmQF8dSKGA6tqzQq+0ZI3xkjyQj46I4/M0K2GbMpcAFOcbr3A==", "dev": true, "funding": [ { @@ -1540,17 +1530,6 @@ "color-support": "bin.js" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/commander": { "version": "2.17.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", @@ -1704,14 +1683,6 @@ "node": ">=10" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -1800,9 +1771,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.427", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.427.tgz", - "integrity": "sha512-HK3r9l+Jm8dYAm1ctXEWIC+hV60zfcjS9UA5BDlYvnI5S7PU/yytjpvSrTNrSSRRkuu3tDyZhdkwIczh+0DWaw==", + "version": "1.4.435", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.435.tgz", + "integrity": "sha512-B0CBWVFhvoQCW/XtjRzgrmqcgVWg6RXOEM/dK59+wFV93BFGR6AeNKc4OyhM+T3IhJaOOG8o/V+33Y2mwJWtzw==", "dev": true }, "node_modules/emoji-regex": { @@ -1859,15 +1830,15 @@ } }, "node_modules/eslint": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.42.0.tgz", - "integrity": "sha512-ulg9Ms6E1WPf67PHaEY4/6E2tEn5/f7FXGzr3t9cBMugOmf1INYvuUwwh1aXQN4MfJ6a5K2iNwP3w4AColvI9A==", + "version": "8.43.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.43.0.tgz", + "integrity": "sha512-aaCpf2JqqKesMFGgmRPessmVKjcGXqdlAYLLC3THM8t5nBRZRQ+st5WM/hoJXkdioEXLLbXgclUpM0TXo5HX5Q==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.4.0", "@eslint/eslintrc": "^2.0.3", - "@eslint/js": "8.42.0", + "@eslint/js": "8.43.0", "@humanwhocodes/config-array": "^0.11.10", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -2268,25 +2239,6 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, - "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -2300,19 +2252,6 @@ "node": ">=8.0.0" } }, - "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/form-data-encoder": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", @@ -3272,17 +3211,6 @@ "node": ">= 0.6" } }, - "node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/mimic-response": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", @@ -3617,12 +3545,6 @@ "node": ">=0.10.0" } }, - "node_modules/node-linux-pam/node_modules/aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, "node_modules/node-linux-pam/node_modules/are-we-there-yet": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", @@ -4676,11 +4598,6 @@ "node": ">=8" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -4966,9 +4883,9 @@ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "node_modules/semver": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", + "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", "optional": true, "dependencies": { "lru-cache": "^6.0.0" @@ -5086,13 +5003,13 @@ } }, "node_modules/sinon": { - "version": "15.1.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.1.0.tgz", - "integrity": "sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==", + "version": "15.2.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.2.0.tgz", + "integrity": "sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw==", "dev": true, "dependencies": { "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^10.2.0", + "@sinonjs/fake-timers": "^10.3.0", "@sinonjs/samsam": "^8.0.0", "diff": "^5.1.0", "nise": "^5.1.4", @@ -6057,9 +5974,9 @@ } }, "@eslint/js": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.42.0.tgz", - "integrity": "sha512-6SWlXpWU5AvId8Ac7zjzmIOqMOba/JWY8XZ4A7q7Gn1Vlfg/SFFIlrtHXt9nPn4op9ZPAkl91Jao+QQv3r/ukw==", + "version": "8.43.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.43.0.tgz", + "integrity": "sha512-s2UHCoiXfxMvmfzqoN+vrQ84ahUSYde9qNO1MdxmoEhyHWsfmwOpFlwYV+ePJEVc7gFnATGUi376WowX1N7tFg==", "dev": true }, "@humanwhocodes/config-array": { @@ -6198,9 +6115,9 @@ "dev": true }, "@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", "dev": true }, "@jridgewell/trace-mapping": { @@ -6211,14 +6128,6 @@ "requires": { "@jridgewell/resolve-uri": "3.1.0", "@jridgewell/sourcemap-codec": "1.4.14" - }, - "dependencies": { - "@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", - "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", - "dev": true - } } }, "@mapbox/node-pre-gyp": { @@ -6285,9 +6194,9 @@ } }, "@sinonjs/fake-timers": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.2.0.tgz", - "integrity": "sha512-OPwQlEdg40HAj5KNF8WW6q2KG4Z+cBCZb3m4ninfTZKaBmbIJodviQsDBoYMPHkOyJJMHnOJo5j2+LKDOhOACg==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", "dev": true, "requires": { "@sinonjs/commons": "^3.0.0" @@ -6400,9 +6309,9 @@ "optional": true }, "acorn": { - "version": "8.8.2", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", - "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.9.0.tgz", + "integrity": "sha512-jaVNAFBHNLXspO543WnNNPZFRtavh3skAkITqD0/2aeMkKZTN+254PyhwxFYrk3vQ1xfY+2wbesJMs/JC8/PwQ==", "dev": true }, "acorn-jsx": { @@ -6489,9 +6398,9 @@ } }, "aproba": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", - "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "optional": true }, "archy": { @@ -6551,21 +6460,6 @@ "integrity": "sha512-5lNGRB5g5i2bGIzb+J1QQE1iKU/WEMVBReFIc5pPDWjcPj23otPL0eI6PB2v7QPi0qU6Mhym5D3y0ZiSIOf3GA==", "optional": true }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" - }, - "axios": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", - "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", - "requires": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -6653,13 +6547,13 @@ "dev": true }, "browserslist": { - "version": "4.21.7", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.7.tgz", - "integrity": "sha512-BauCXrQ7I2ftSqd2mvKHGo85XR0u7Ru3C/Hxsy/0TkfCtjrmAbPdzLGasmoiBxplpDXlPvdjX9u7srIMfgasNA==", + "version": "4.21.9", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.9.tgz", + "integrity": "sha512-M0MFoZzbUrRU4KNfCrDLnvyE7gub+peetoTid3TBIqtunaDJyXlwhakT+/VkvSXcfIzFfK/nkCs4nmyTmxdNSg==", "dev": true, "requires": { - "caniuse-lite": "^1.0.30001489", - "electron-to-chromium": "^1.4.411", + "caniuse-lite": "^1.0.30001503", + "electron-to-chromium": "^1.4.431", "node-releases": "^2.0.12", "update-browserslist-db": "^1.0.11" } @@ -6692,17 +6586,24 @@ "integrity": "sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==" }, "cacheable-request": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.10.tgz", - "integrity": "sha512-v6WB+Epm/qO4Hdlio/sfUn69r5Shgh39SsE9DSd4bIezP0mblOlObI+I0kUEM7J0JFc+I7pSeMeYaOYtX1N/VQ==", + "version": "10.2.11", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.11.tgz", + "integrity": "sha512-kn0t0oJnlFo1Nzl/AYQzS/oByMtmaqLasFUa7MUMsiTrIHy8TxSkx2KzWCybE3Nuz1F4sJRGnLAfUGsPe47viQ==", "requires": { "@types/http-cache-semantics": "^4.0.1", - "get-stream": "^6.0.1", + "get-stream": "^7.0.0", "http-cache-semantics": "^4.1.1", "keyv": "^4.5.2", "mimic-response": "^4.0.0", "normalize-url": "^8.0.0", "responselike": "^3.0.0" + }, + "dependencies": { + "get-stream": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-7.0.0.tgz", + "integrity": "sha512-ql6FW5b8tgMYvI4UaoxG3EQN3VyZ6VeQpxNBGg5BZ4xD4u+HJeprzhMMA4OCBEGQgSR+m87pstWMpiVW64W8Fw==" + } } }, "caching-transform": { @@ -6740,9 +6641,9 @@ "devOptional": true }, "caniuse-lite": { - "version": "1.0.30001498", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001498.tgz", - "integrity": "sha512-LFInN2zAwx3ANrGCDZ5AKKJroHqNKyjXitdV5zRIVIaQlXKj3GmxUKagoKsjqUfckpAObPCEWnk5EeMlyMWcgw==", + "version": "1.0.30001505", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001505.tgz", + "integrity": "sha512-jaAOR5zVtxHfL0NjZyflVTtXm3D3J9P15zSJ7HmQF8dSKGA6tqzQq+0ZI3xkjyQj46I4/M0K2GbMpcAFOcbr3A==", "dev": true }, "chalk": { @@ -6841,14 +6742,6 @@ "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", "optional": true }, - "combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, "commander": { "version": "2.17.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.17.1.tgz", @@ -6963,11 +6856,6 @@ "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==" }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" - }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -7029,9 +6917,9 @@ } }, "electron-to-chromium": { - "version": "1.4.427", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.427.tgz", - "integrity": "sha512-HK3r9l+Jm8dYAm1ctXEWIC+hV60zfcjS9UA5BDlYvnI5S7PU/yytjpvSrTNrSSRRkuu3tDyZhdkwIczh+0DWaw==", + "version": "1.4.435", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.435.tgz", + "integrity": "sha512-B0CBWVFhvoQCW/XtjRzgrmqcgVWg6RXOEM/dK59+wFV93BFGR6AeNKc4OyhM+T3IhJaOOG8o/V+33Y2mwJWtzw==", "dev": true }, "emoji-regex": { @@ -7073,15 +6961,15 @@ "dev": true }, "eslint": { - "version": "8.42.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.42.0.tgz", - "integrity": "sha512-ulg9Ms6E1WPf67PHaEY4/6E2tEn5/f7FXGzr3t9cBMugOmf1INYvuUwwh1aXQN4MfJ6a5K2iNwP3w4AColvI9A==", + "version": "8.43.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.43.0.tgz", + "integrity": "sha512-aaCpf2JqqKesMFGgmRPessmVKjcGXqdlAYLLC3THM8t5nBRZRQ+st5WM/hoJXkdioEXLLbXgclUpM0TXo5HX5Q==", "dev": true, "requires": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.4.0", "@eslint/eslintrc": "^2.0.3", - "@eslint/js": "8.42.0", + "@eslint/js": "8.43.0", "@humanwhocodes/config-array": "^0.11.10", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", @@ -7370,11 +7258,6 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, - "follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" - }, "foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -7385,16 +7268,6 @@ "signal-exit": "^3.0.2" } }, - "form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - }, "form-data-encoder": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", @@ -8099,14 +7972,6 @@ "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, - "mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "requires": { - "mime-db": "1.52.0" - } - }, "mimic-response": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", @@ -8364,12 +8229,6 @@ "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", "optional": true }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, "are-we-there-yet": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", @@ -9204,11 +9063,6 @@ "fromentries": "^1.2.0" } }, - "proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -9407,9 +9261,9 @@ "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "semver": { - "version": "7.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.1.tgz", - "integrity": "sha512-Wvss5ivl8TMRZXXESstBA4uR5iXgEN/VC5/sOcuXdVLzcdkz4HWetIoRfG5gb5X+ij/G9rw9YoGn3QoQ8OCSpw==", + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.2.tgz", + "integrity": "sha512-SoftuTROv/cRjCze/scjGyiDtcUyxw1rgYQSZY7XTmtR5hX+dm76iDbTH8TkLPHCQmlbQVSSbNZCPM2hb0knnQ==", "optional": true, "requires": { "lru-cache": "^6.0.0" @@ -9486,13 +9340,13 @@ } }, "sinon": { - "version": "15.1.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.1.0.tgz", - "integrity": "sha512-cS5FgpDdE9/zx7no8bxROHymSlPLZzq0ChbbLk1DrxBfc+eTeBK3y8nIL+nu/0QeYydhhbLIr7ecHJpywjQaoQ==", + "version": "15.2.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-15.2.0.tgz", + "integrity": "sha512-nPS85arNqwBXaIsFCkolHjGIkFo+Oxu9vbgmBJizLAhqe6P2o3Qmj3KCUoRkfhHtvgDhZdWD3risLHAUJ8npjw==", "dev": true, "requires": { "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^10.2.0", + "@sinonjs/fake-timers": "^10.3.0", "@sinonjs/samsam": "^8.0.0", "diff": "^5.1.0", "nise": "^5.1.4", diff --git a/package.json b/package.json index c28c64d..a950ea0 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,8 @@ "@squeep/html-template-helper": "git+https://git.squeep.com/squeep-html-template-helper#v1.4.0", "@squeep/logger-json-console": "^2.0.1", "@squeep/web-linking": "^1.0.8", - "axios": "^1.4.0", "feedparser": "^2.2.10", + "got": "^13.0.0", "htmlparser2": "^9.0.0", "iconv": "^3.0.1" }, @@ -47,7 +47,7 @@ "pg-promise": "^11.5.0" }, "devDependencies": { - "eslint": "^8.42.0", + "eslint": "^8.43.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^6.1.1", "eslint-plugin-security": "^1.7.1", @@ -57,6 +57,6 @@ "mocha-steps": "^1.3.0", "nyc": "^15.1.0", "pre-commit": "^1.2.2", - "sinon": "^15.1.0" + "sinon": "^15.2.0" } } diff --git a/server.js b/server.js index c12d19a..d5b206c 100644 --- a/server.js +++ b/server.js @@ -17,7 +17,9 @@ const ADDR = process.env.LISTEN_ADDR || '127.0.0.1'; (async function main () { let config, logger, db, service; try { + // This provides an async context store for persisting unique log data per request, id est a requestId. const asyncLocalStorage = new AsyncLocalStorage(); + config = new Config(process.env.NODE_ENV); logger = new Logger(config.logger, { nodeId: config.nodeId }, asyncLocalStorage); db = new DB(logger, config); @@ -26,7 +28,7 @@ const ADDR = process.env.LISTEN_ADDR || '127.0.0.1'; http.createServer(async (req, res) => { await asyncLocalStorage.run({}, async () => { - await service.dispatch(req, res); + return service.dispatch(req, res); }); }).listen(PORT, ADDR, (err) => { if (err) { diff --git a/src/common.js b/src/common.js index 4ed310f..f7c54ef 100644 --- a/src/common.js +++ b/src/common.js @@ -61,20 +61,31 @@ const freezeDeep = (o) => { /** - * Pick out useful axios response fields. + * Pick out useful got response fields. * @param {*} res * @returns */ -const axiosResponseLogData = (res) => { +const gotResponseLogData = (res) => { const data = common.pick(res, [ - 'status', - 'statusText', + 'statusCode', + 'statusMessage', 'headers', - 'elapsedTimeMs', - 'data', + 'body', + 'error', ]); - if (data.data) { - data.data = logTruncate(data.data, 100); + if (typeof res.body === 'string') { + data.body = logTruncate(data.body, 100); + } else if (res.body instanceof Buffer) { + data.body = ``; + } + if (res?.timings?.phases?.total) { + data.elapsedTimeMs = res.timings.phases.total; + } + if (res?.redirectUrls?.length) { + data.redirectUrls = res.redirectUrls; + } + if (res?.retryCount) { + data.retryCount = res.retryCount; } return data; }; @@ -155,7 +166,7 @@ module.exports = { ...common, arrayChunk, attemptRetrySeconds, - axiosResponseLogData, + gotResponseLogData, ensureArray, freezeDeep, logTruncate, diff --git a/src/communication.js b/src/communication.js index feda588..c0d08fd 100644 --- a/src/communication.js +++ b/src/communication.js @@ -5,7 +5,6 @@ * worker which initiates most of them. */ -const axios = require('axios'); const common = require('./common'); const crypto = require('crypto'); const Enum = require('./enum'); @@ -14,8 +13,6 @@ const Worker = require('./worker'); const LinkHelper = require('./link-helper'); const { version: packageVersion, name: packageName } = require('../package.json'); // For default UA string -const { performance } = require('perf_hooks'); - const _fileScope = common.fileScope(__filename); class Communication { @@ -32,28 +29,68 @@ class Communication { this.logger.error(_fileScope('constructor'), 'empty dingus.selfBaseUrl value, server responses will not be compliant'); } - // Set common options - this.axios = axios.create({ - validateStatus: null, // Non-success responses are not exceptional - headers: { - [Enum.Header.UserAgent]: Communication.userAgentString(options.userAgent), - }, - }); - - this.axios.interceptors.request.use((request) => { - request.startTimestampMs = performance.now(); - return request; - }); - this.axios.interceptors.response.use((response) => { - response.elapsedTimeMs = performance.now() - response.config.startTimestampMs; - return response; - }); + this.Got = undefined; // Will become the async imported got. + this.got = this._init; // First invocation imports got and replaces this. this.worker = new Worker(logger, db, this.workFeed.bind(this), options); this.worker.start(); } + /** + * Do a little dance to cope with ESM dynamic import. + * @param {...any} args + * @returns {Promise} + */ + async _init(...args) { + if (!this.Got) { + // For some reason eslint is confused about import being supported here. + // eslint-disable-next-line + this.Got = await import('got'); + this.got = this.Got.got.extend({ + followRedirect: false, // Outgoing API calls should not encounter redirects + throwHttpErrors: false, // We will be checking status codes explicitly + headers: { + [Enum.Header.UserAgent]: Communication.userAgentString(this.options.userAgent), + }, + timeout: { + request: this.options.communication.requestTimeoutMs || 120000, + }, + hooks: { + beforeRetry: [ + this._onRetry, + ], + }, + }); + } + + /* istanbul ignore if */ + if (args.length) { + /* istanbul ignore next */ + return this.got(...args); + } + } + + + /** + * Take note of transient retries. + * @param {*} error + * @param {*} retryCount + */ + _onRetry(error, retryCount) { + const _scope = _fileScope('_onRetry'); + this.logger.debug(_scope, 'retry', { retryCount, error }); + } + + + /** + * Construct a user-agent value. + * @param {Object} userAgentConfig + * @param {String=} userAgentConfig.product + * @param {String=} userAgentConfig.version + * @param {String=} userAgentConfig.implementation + * @returns {String} + */ static userAgentString(userAgentConfig) { // eslint-disable-next-line security/detect-object-injection const _conf = (field, def) => (userAgentConfig && field in userAgentConfig) ? userAgentConfig[field] : def; @@ -70,7 +107,7 @@ class Communication { /** * Generate a random string. * @param {Integer} bytes - * @returns {String} + * @returns {Promise} */ static async generateChallenge(bytes = 30) { return (await common.randomBytesAsync(bytes)).toString('base64'); @@ -85,9 +122,10 @@ class Communication { * @returns {String} */ static signature(message, secret, algorithm) { - const hmac = crypto.createHmac(algorithm, secret); - hmac.update(message); - return `${algorithm}=${hmac.digest('hex')}`; + const hmac = crypto.createHmac(algorithm, secret) + .update(message) + .digest('hex'); + return `${algorithm}=${hmac}`; } @@ -95,110 +133,12 @@ class Communication { * Generate the hash for content. * @param {Buffer} content * @param {String} algorithm - * @returns - */ - static contentHash(content, algorithm) { - const hash = crypto.createHash(algorithm); - hash.update(content); - return hash.digest('hex'); - } - - - /** - * A request skeleton config. - * @param {String} method - * @param {String} requestUrl - * @param {String} body - * @param {Object} params - */ - static _axiosConfig(method, requestUrl, body, params = {}, headers = {}) { - const urlObj = new URL(requestUrl); - const config = { - method, - url: `${urlObj.origin}${urlObj.pathname}`, - params: urlObj.searchParams, - headers, - ...(body && { data: body }), - // Setting this does not appear to be enough to keep axios from parsing JSON response into object - responseType: 'text', - // So force the matter by eliding all response transformations - transformResponse: [ (res) => res ], - }; - Object.entries(params).map(([k, v]) => config.params.set(k, v)); - return config; - } - - - /** - * Create request config for verifying an intent. - * @param {URL} requestUrl - * @param {String} topicUrl - * @param {String} mode - * @param {Integer} leaseSeconds - * @param {String} challenge - */ - static _intentVerifyAxiosConfig(requestUrl, topicUrl, mode, leaseSeconds, challenge) { - // Explicitly convert leaseSeconds to string, due to some DB backends. (Looking at you, sqlite..) - leaseSeconds = leaseSeconds.toString(); - - return Communication._axiosConfig('GET', requestUrl, undefined, { - 'hub.mode': mode, - 'hub.topic': topicUrl, - 'hub.challenge': challenge, - 'hub.lease_seconds': leaseSeconds, - }, {}); - } - - - /** - * Create request config for denying an intent. - * @param {String} requestUrl - * @param {String} topicUrl - * @param {String} reason - * @returns {String} - */ - static _intentDenyAxiosConfig(requestUrl, topicUrl, reason) { - return Communication._axiosConfig('GET', requestUrl, undefined, { - 'hub.mode': Enum.Mode.Denied, - 'hub.topic': topicUrl, - ...(reason && { 'hub.reason': reason }), - }, {}); - } - - - /** - * Create request config for querying publisher for subscription validation. - * @param {Topic} topic - * @param {Verification} verification - * @returns {String} - */ - static _publisherValidationAxiosConfig(topic, verification) { - const body = { - callback: verification.callback, - topic: topic.url, - ...(verification.httpFrom && { from: verification.httpFrom }), - ...(verification.httpRemoteAddr && { address: verification.httpRemoteAddr }), - }; - return Communication._axiosConfig('POST', topic.publisherValidationUrl, body, {}, { - [Enum.Header.ContentType]: Enum.ContentType.ApplicationJson, - }); - } - - - /** - * Create request config for fetching topic content. - * Prefer existing content-type, but accept anything. - * @param {Topic} topic * @returns {String} */ - static _topicFetchAxiosConfig(topic) { - const acceptWildcard = '*/*' + (topic.contentType ? ';q=0.9' : ''); - const acceptPreferred = [topic.contentType, acceptWildcard].filter((x) => x).join(', '); - return Communication._axiosConfig('GET', topic.url, undefined, {}, { - [Enum.Header.Accept]: acceptPreferred, - ...(topic.httpEtag && { [Enum.Header.IfNoneMatch]: topic.httpEtag }), - ...(topic.httpLastModified && { [Enum.Header.IfModifiedSince]: topic.httpLastModified }), - }); + static contentHash(content, algorithm) { + return crypto.createHash(algorithm) + .update(content) + .digest('hex'); } @@ -256,19 +196,37 @@ class Communication { } } - const u = new URL(verification.callback); - let callbackRequestConfig, challenge; + const callbackRequestConfig = { + method: 'GET', + url: new URL(verification.callback), + responseType: 'text', + }; + const callbackParams = { + 'hub.topic': topic.url, + 'hub.mode': verification.mode, + }; + + let challenge; if (verification.mode === Enum.Mode.Denied) { - // Denials don't have a challenge. - callbackRequestConfig = Communication._intentDenyAxiosConfig(u, topic.url, verification.reason); + // Denials don't have a challenge, but might have a reason. + if (verification.reason) { + callbackParams['hub.reason'] = verification.reason; + } } else { // Subscriptions and unsubscriptions require challenge matching. challenge = await Communication.generateChallenge(); - callbackRequestConfig = Communication._intentVerifyAxiosConfig(u, topic.url, verification.mode, verification.leaseSeconds, challenge); + Object.assign(callbackParams, { + 'hub.challenge': challenge, + // Explicitly convert leaseSeconds to string, due to some DB backends. (Looking at you, sqlite..) + 'hub.lease_seconds': verification.leaseSeconds.toString(), + }); } + Object.entries(callbackParams) + .forEach(([k, v]) => callbackRequestConfig.url.searchParams.set(k, v)) + ; const logInfoData = { - callbackUrl: u.href, + callbackUrl: callbackRequestConfig.url.href, topicUrl: topic.url, mode: verification.mode, originalRequestId: verification.requestId, @@ -280,18 +238,18 @@ class Communication { let response; try { - response = await this.axios(callbackRequestConfig); + response = await this.got(callbackRequestConfig); } catch (e) { this.logger.error(_scope, 'verification request failed', { ...logInfoData, error: e }); await this.db.verificationIncomplete(dbCtx, verificationId, this.options.communication.retryBackoffSeconds); return; } - logInfoData.response = common.axiosResponseLogData(response); + logInfoData.response = common.gotResponseLogData(response); this.logger.debug(_scope, 'verification response', logInfoData ); let verificationAccepted = true; // Presume success. - switch (common.httpStatusCodeClass(response.status)) { + switch (common.httpStatusCodeClass(response.statusCode)) { case 2: // Success, fall out of switch. break; @@ -315,7 +273,7 @@ class Communication { } if ([Enum.Mode.Subscribe, Enum.Mode.Unsubscribe].includes(verification.mode) - && response.data !== challenge) { + && response.body !== challenge) { this.logger.info(_scope, 'verification rejected by challenge', logInfoData); verificationAccepted = false; } @@ -341,7 +299,7 @@ class Communication { case Enum.Mode.Denied: await this.db.subscriptionDelete(txCtx, verification.callback, verification.topicId); if (topic.isDeleted) { - // Remove a deleted topic after he last subscription is notified. + // Remove a deleted topic after the last subscription is notified. await this.db.topicPendingDelete(txCtx, topic.id); } break; @@ -363,6 +321,8 @@ class Communication { * Updates (and persists) verification. * Returns boolean of status of publisher contact, and hence * whether to continue verification with client. + * + * This is not defined by the spec. We opt to speak JSON here. * @param {*} dbCtx * @param {TopicData} topic * @param {VerificationData} verification @@ -370,7 +330,6 @@ class Communication { */ async publisherValidate(dbCtx, topic, verification) { const _scope = _fileScope('publisherValidate'); - const publisherValidationRequestConfig = Communication._publisherValidationAxiosConfig(topic, verification); const logInfoData = { topicUrl: topic.url, callbackUrl: verification.callback, @@ -380,18 +339,29 @@ class Communication { this.logger.info(_scope, 'publisher validation request', logInfoData); + const publisherValidationRequestConfig = { + method: 'POST', + url: topic.publisherValidationUrl, + json: { + callback: verification.callback, + topic: topic.url, + ...(verification.httpFrom && { from: verification.httpFrom }), + ...(verification.httpRemoteAddr && { address: verification.httpRemoteAddr }), + }, + responseType: 'json', + }; try { - response = await this.axios(publisherValidationRequestConfig); + response = await this.got(publisherValidationRequestConfig); } catch (e) { this.logger.error(_scope, 'publisher validation failed', { ...logInfoData, error: e }); return false; // Do not continue with client verification. } - logInfoData.response = common.axiosResponseLogData(response); + logInfoData.response = common.gotResponseLogData(response); this.logger.debug(_scope, 'validation response', logInfoData); let verificationNeedsUpdate = false; - switch (common.httpStatusCodeClass(response.status)) { + switch (common.httpStatusCodeClass(response.statusCode)) { case 2: this.logger.info(_scope, 'publisher validation complete, allowed', logInfoData); break; @@ -452,22 +422,32 @@ class Communication { return; } - const updateRequestConfig = Communication._topicFetchAxiosConfig(topic); + const updateRequestConfig = { + followRedirect: true, + method: 'GET', + url: topic.url, + headers: { + [Enum.Header.Accept]: [topic.contentType, `*/*${topic.contentType ? ';q=0.9' : ''}`].filter((x) => x).join(', '), + ...(topic.httpEtag && { [Enum.Header.IfNoneMatch]: topic.httpEtag }), + ...(topic.httpLastModified && { [Enum.Header.IfModifiedSince]: topic.httpLastModified }), + }, + responseType: 'buffer', + }; this.logger.info(_scope, 'topic update request', logInfoData); let response; try { - response = await this.axios(updateRequestConfig); + response = await this.got(updateRequestConfig); } catch (e) { this.logger.error(_scope, 'update request failed', logInfoData); await this.db.topicFetchIncomplete(dbCtx, topicId, this.options.communication.retryBackoffSeconds); return; } - logInfoData.response = common.axiosResponseLogData(response); + logInfoData.response = common.gotResponseLogData(response); this.logger.debug(_scope, 'fetch response', logInfoData); - switch (common.httpStatusCodeClass(response.status)) { + switch (common.httpStatusCodeClass(response.statusCode)) { case 2: case 3: // Fall out of switch on success @@ -484,13 +464,13 @@ class Communication { return; } - if (response.status === 304) { + if (response.statusCode === 304) { this.logger.info(_scope, 'content has not changed, per server', logInfoData); await this.db.topicFetchComplete(dbCtx, topicId); return; } - const contentHash = Communication.contentHash(response.data, topic.contentHashAlgorithm); + const contentHash = Communication.contentHash(response.body, topic.contentHashAlgorithm); logInfoData.contentHash = contentHash; if (topic.contentHash === contentHash) { this.logger.info(_scope, 'content has not changed', logInfoData); @@ -498,7 +478,7 @@ class Communication { return; } - const validHub = await this.linkHelper.validHub(topic.url, response.headers, response.data); + const validHub = await this.linkHelper.validHub(topic.url, response.headers, response.body); if (!validHub) { this.logger.info(_scope, 'retrieved topic does not list us as hub', { logInfoData }); if (this.options.communication.strictTopicHubLink) { @@ -520,7 +500,7 @@ class Communication { await this.db.transaction(dbCtx, async (txCtx) => { await this.db.topicSetContent(txCtx, { topicId, - content: Buffer.from(response.data), + content: Buffer.from(response.body), contentHash, ...(contentType && { contentType }), ...(httpETag && { httpETag }), @@ -587,26 +567,32 @@ class Communication { logInfoData.contentLength = topic.content.length; logInfoData.contentHash = topic.contentHash; - const updateAxiosConfig = Communication._axiosConfig('POST', subscription.callback, topic.content, {}, { - [Enum.Header.Link]: `<${topic.url}>; rel="self"${this.linkHub}`, - [Enum.Header.ContentType]: topic.contentType || Enum.ContentType.TextPlain, - ...(subscription.secret && { [Enum.Header.XHubSignature]: Communication.signature(topic.content, subscription.secret, subscription.signatureAlgorithm) }), - }); + const updateConfig = { + method: 'POST', + url: subscription.callback, + body: topic.content, + headers: { + [Enum.Header.Link]: `<${topic.url}>; rel="self"${this.linkHub}`, + [Enum.Header.ContentType]: topic.contentType || Enum.ContentType.TextPlain, + ...(subscription.secret && { [Enum.Header.XHubSignature]: Communication.signature(topic.content, subscription.secret, subscription.signatureAlgorithm) }), + }, + responseType: 'text', + }; this.logger.info(_scope, 'update request', logInfoData); let response; try { - response = await this.axios(updateAxiosConfig); + response = await this.got(updateConfig); } catch (e) { this.logger.error(_scope, 'update request failed', { ...logInfoData, error: e }); await this.db.subscriptionDeliveryIncomplete(dbCtx, subscription.callback, subscription.topicId, this.options.communication.retryBackoffSeconds); return; } - logInfoData.response = common.axiosResponseLogData(response); + logInfoData.response = common.gotResponseLogData(response); this.logger.debug(_scope, 'update response', logInfoData); - switch (common.httpStatusCodeClass(response.status)) { + switch (common.httpStatusCodeClass(response.statusCode)) { case 2: // Fall out of switch on success. break; @@ -617,7 +603,7 @@ class Communication { return; case 4: - if (response.status === 410) { // GONE + if (response.statusCode === 410) { // GONE this.logger.info(_scope, 'client declined further updates', logInfoData); await this.db.subscriptionDeliveryGone(dbCtx, subscription.callback, subscription.topicId); return; diff --git a/src/db/postgres/index.js b/src/db/postgres/index.js index e7a006c..b07b678 100644 --- a/src/db/postgres/index.js +++ b/src/db/postgres/index.js @@ -68,6 +68,8 @@ class DatabasePostgres extends Database { // Log errors pgpInitOptions.error = (err, event) => { this.logger.error(_fileScope('pgp:error'), '', { err, event }); + + // TODO: close connection on err.code === '57P03' database shutting down }; // Deophidiate column names in-place, log results diff --git a/src/manager.js b/src/manager.js index 74730c2..d33d345 100644 --- a/src/manager.js +++ b/src/manager.js @@ -354,7 +354,7 @@ class Manager { // Parse and validate all the topics in the request. data.publishTopics = await this._publishTopics(dbCtx, data, requestId); - if (!data.publishTopics || !data.publishTopics.length) { + if (!data?.publishTopics?.length) { const details = Manager._prettyDetails(['no valid topic urls to publish'], []); throw new ResponseError(Enum.ErrorResponse.BadRequest, details); } @@ -636,7 +636,7 @@ class Manager { this.logger.debug(_scope, 'got topics', { topics: ctx.topics }); // Profile users can only see related topics. - if (ctx.session && ctx.session.authenticatedProfile) { + if (ctx?.session?.authenticatedProfile) { const profileUrlObj = new URL(ctx.session.authenticatedProfile); ctx.topics = ctx.topics.filter((topic) => { const topicUrlObj = new URL(topic.url); @@ -673,7 +673,7 @@ class Manager { this.logger.debug(_scope, 'got topic details', { topic: ctx.topic, subscriptions: ctx.subscriptions, updates: ctx.publishCount }); // Profile users can only see related topics. - if (ctx.session && ctx.session.authenticatedProfile) { + if (ctx?.session?.authenticatedProfile) { const profileUrlObj = new URL(ctx.session.authenticatedProfile); const topicUrlObj = new URL(ctx.topic.url); if (!Manager._profileControlsTopic(profileUrlObj, topicUrlObj)) { @@ -683,7 +683,7 @@ class Manager { } res.end(Template.adminTopicDetailsHTML(ctx, this.options)); - this.logger.info(_scope, 'finished', { ctx, subscriptions: ctx.subscriptions.length, topic: ctx.topic && ctx.topic.id || ctx.topic }); + this.logger.info(_scope, 'finished', { ctx, subscriptions: ctx.subscriptions.length, topic: ctx?.topic?.id || ctx.topic }); } diff --git a/src/service.js b/src/service.js index b498829..d77d06c 100644 --- a/src/service.js +++ b/src/service.js @@ -72,12 +72,19 @@ class Service extends Dingus { /** * Rearrange logging data. + * @param {http.ClientRequest} req + * @param {http.ServerResponse} res + * @param {Object} ctx */ async preHandler(req, res, ctx) { await super.preHandler(req, res, ctx); const logObject = this.asyncLocalStorage.getStore(); - logObject.requestId = ctx.requestId; - delete ctx.requestId; + // FIXME: for some reason, returning from the super.preHandler loses async context? + // Workaround until cause and solution are found. + if (logObject) { + logObject.requestId = ctx.requestId; + delete ctx.requestId; + } } diff --git a/test-e2e/fake-servers-client.js b/test-e2e/fake-servers-client.js index 7dfea3b..255fa3f 100644 --- a/test-e2e/fake-servers-client.js +++ b/test-e2e/fake-servers-client.js @@ -4,20 +4,30 @@ * Wrapper interface for controlling fake-servers. */ -const axios = require('axios'); - class FakeClient { constructor(host, subscriberPort, topicPort) { this.logger = console; this.host = host; this.subscriberPort = subscriberPort; this.topicPort = topicPort; - this.axios = axios.create({ - validateStatus: (statusCode) => (Math.floor(statusCode / 100)) === 2, - headers: { - 'User-Agent': 'FakeClient', - }, - }); + this.Got = undefined; + this.got = this._init.bind(this); + } + + async _init(...args) { + if (!this.Got) { + // eslint-disable-next-line + this.Got = await import('got'); + this.got = this.Got.got.extend({ + headers: { + 'User-Agent': 'FakeClient', + }, + responseType: 'text', + }); + } + if (args.length) { + return this.got(...args); + } } topicUrl(id) { @@ -28,42 +38,43 @@ class FakeClient { return `http://${this.host}:${this.subscriberPort}/subscriber/${id}${extra}`; } - static _axiosRequestConfig(method, url, params = {}, headers = {}, data) { - const urlObj = new URL(url); - const config = { + static _requestConfig(method, url, params = {}, headers = {}, body = undefined) { + const gotConfig = { method, - url: `${urlObj.origin}${urlObj.pathname}`, - params: urlObj.searchParams, + url: new URL(url), headers, - ...(data && { data }), - responseType: 'text', - transformResponse: [ (res) => res ], + ...(body && { body }), }; - Object.entries(params).map(([k, v]) => config.params.set(k, v)); - return config; + Object.entries(params).forEach(([k, v]) => gotConfig.url.searchParams.set(k, v)); + return gotConfig; + } + + static _formData(obj) { + return Object.entries(obj) + .map((entry) => entry.map(encodeURIComponent).join('=')) + .join('&') + ; } async subscribe(hubUrl, subscriberId, topicId, postData = {}) { const topicUrl = this.topicUrl(topicId); const subscriberUrl = this.subscriberUrl(subscriberId); - const data = { + const data = FakeClient._formData({ 'hub.callback': subscriberUrl, 'hub.mode': 'subscribe', 'hub.topic': topicUrl, 'hub.lease_seconds': 60, 'hub.secret': 'sharedSecret', ...postData, - }; - const formData = new URLSearchParams(data).toString(); - const headers = { + }); + const config = FakeClient._requestConfig('POST', hubUrl, {}, { 'Content-Type': 'application/x-www-form-urlencoded', - }; - + }, data); try { - return this.axios(FakeClient._axiosRequestConfig('POST', hubUrl, {}, headers, formData)); - } catch (e) { - this.logger.error('subscribe', e); - throw e; + return await this.got(config); + } catch (error) { + this.logger.error('subscribe', error, config); + throw error; } } @@ -83,14 +94,15 @@ class FakeClient { contentType: 'text/plain', }; const url = this.topicUrl(id); + const config = FakeClient._requestConfig('PUT', url, { + ...defaultBehavior, + ...behavior, + }); try { - return this.axios(FakeClient._axiosRequestConfig('PUT', url, { - ...defaultBehavior, - ...behavior, - })); - } catch (e) { - this.logger.error('topicSet', e); - throw e; + return await this.got(config); + } catch (error) { + this.logger.error('topicSet', error, config); + throw error; } } @@ -99,12 +111,13 @@ class FakeClient { * @param {String} id */ async topicDelete(id) { - const url =this.topicUrl(id); + const url = this.topicUrl(id); + const config = FakeClient._requestConfig('DELETE', url); try { - return this.axios(FakeClient._axiosRequestConfig('DELETE', url)); - } catch (e) { - this.logger.error('topicDelete', e); - throw e; + return await this.got(config); + } catch (error) { + this.logger.error('topicDelete', error, config); + throw error; } } @@ -121,14 +134,15 @@ class FakeClient { matchChallenge: true, }; const url = this.subscriberUrl(id, '/verify'); + const config = FakeClient._requestConfig('PUT', url, { + ...defaultBehavior, + ...behavior, + }); try { - return this.axios(FakeClient._axiosRequestConfig('PUT', url, { - ...defaultBehavior, - ...behavior, - })); - } catch (e) { - this.logger.error('subscriberSetVerify', e); - throw e; + return await this.got(config); + } catch (error) { + this.logger.error('subscriberSetVerify', error, config); + throw error; } } @@ -138,19 +152,20 @@ class FakeClient { * @param {Object} behavior * @param {Number} behavior.statusCode */ - async subscriberSetContent(id, behavior = {}) { - const defaultBehavior = { - statusCode: 200, - }; + async subscriberSetContent(id, behavior = {}) { + const defaultBehavior = { + statusCode: 200, + }; const url = this.subscriberUrl(id, '/content'); + const config = FakeClient._requestConfig('PUT', url, { + ...defaultBehavior, + ...behavior, + }); try { - return this.axios(FakeClient._axiosRequestConfig('PUT', url, { - ...defaultBehavior, - ...behavior, - })); - } catch (e) { - this.logger.error('subscriberSetContent', e); - throw e; + return await this.got(config); + } catch (error) { + this.logger.error('subscriberSetContent', error, config); + throw error; } } @@ -160,11 +175,12 @@ class FakeClient { */ async subscriberDelete(id) { const url = this.subscriberUrl(id); + const config = FakeClient._requestConfig('DELETE', url); try { - return this.axios(FakeClient._axiosRequestConfig('DELETE', url)); - } catch (e) { - this.logger.error('subscriberDelete', e); - throw e; + return await this.got(config); + } catch (error) { + this.logger.error('subscriberDelete', error, config); + throw error; } } diff --git a/test-e2e/fake-servers.js b/test-e2e/fake-servers.js index 9d46d48..2071962 100644 --- a/test-e2e/fake-servers.js +++ b/test-e2e/fake-servers.js @@ -46,7 +46,7 @@ class TopicFake extends Dingus { res.setHeader('Link', behavior.selfLink + (behavior.hubLink ? `, ${behavior.hubLink}` : '')); res.statusCode = behavior.statusCode; res.end(behavior.content); - this.logger.info({ method: req.method, statusCode: res.statusCode, url: req.url }); + this.logger.info('TopicFake:getId', { method: req.method, statusCode: res.statusCode, url: req.url }); } async putId(req, res, ctx) { @@ -61,6 +61,7 @@ class TopicFake extends Dingus { }; this.topicBehaviors.set(id, behavior); res.statusCode = 200; + this.logger.info('TopicFake:putId', { method: req.method, statusCode: res.statusCode, url: req.url }); res.end(); } @@ -68,6 +69,7 @@ class TopicFake extends Dingus { this.setResponseType(this.responseTypes, req, res, ctx); this.topicBehaviors.delete(ctx.params.id); res.statusCode = 200; + this.logger.info('TopicFake:deleteId', { method: req.method, statusCode: res.statusCode, url: req.url }); res.end(); } @@ -97,9 +99,9 @@ class SubscriberFake extends Dingus { this.setResponseType(this.responseTypes, req, res, ctx); const behavior = this.verifyBehaviors.get(ctx.params.id); res.statusCode = behavior ? behavior.statusCode : 404; - const response = (behavior && behavior.matchChallenge) ? ctx.queryParams['hub.challenge'] : (behavior && behavior.response); + const response = (behavior?.matchChallenge) ? ctx.queryParams['hub.challenge'] : (behavior?.response); res.end(response); - this.logger.info({ method: req.method, statusCode: res.statusCode, matchChallenge: !!(behavior && behavior.matchChallenge), url: req.url }); + this.logger.info('SubscriberFake:getId', { method: req.method, statusCode: res.statusCode, matchChallenge: !!(behavior?.matchChallenge), url: req.url }); } async postId(req, res, ctx) { @@ -112,7 +114,7 @@ class SubscriberFake extends Dingus { behavior.content = ctx.rawBody; } res.end(); - this.logger.info({ content: behavior && behavior.content, method: req.method, statusCode: res.statusCode, matchChallenge: !!(behavior && behavior.matchChallenge), url: req.url }); + this.logger.info('SubscriberFake:postId', { content: behavior?.content, method: req.method, statusCode: res.statusCode, matchChallenge: !!(behavior?.matchChallenge), url: req.url }); } async putVerify(req, res, ctx) { @@ -129,6 +131,7 @@ class SubscriberFake extends Dingus { } res.statusCode = 200; res.end(); + this.logger.info('SubscriberFake:putVerify', { method: req.method, statusCode: res.statusCode, url: req.url }); } async putContent(req, res, ctx) { @@ -139,6 +142,7 @@ class SubscriberFake extends Dingus { this.contentBehaviors.set(ctx.params.id, behavior); res.statusCode = 200; res.end(); + this.logger.info('SubscriberFake:putContent', { method: req.method, statusCode: res.statusCode, url: req.url }); } async deleteId(req, res, ctx) { @@ -147,6 +151,7 @@ class SubscriberFake extends Dingus { this.verifyBehaviors.delete(ctx.params.id); res.statusCode = 200; res.end(); + this.logger.info('SubscriberFake:deleteId', { method: req.method, statusCode: res.statusCode, url: req.url }); } } // SubscriberFake diff --git a/test-e2e/test-many.js b/test-e2e/test-many.js index 1ff40c9..bc14962 100644 --- a/test-e2e/test-many.js +++ b/test-e2e/test-many.js @@ -1,7 +1,6 @@ 'use strict'; -const assert = require('assert'); -const uuid = require('uuid'); +const crypto = require('crypto'); const FakeServersClient = require('./fake-servers-client'); const subscriberPort = process.env.FAKE_SUBSCRIBER_PORT || 9876; @@ -14,14 +13,14 @@ const hubUrl = `http://${hubAddress}:${hubPort}/`; const client = new FakeServersClient(listenAddress, subscriberPort, topicPort); async function newTopic() { - const id = uuid.v4(); + const id = crypto.randomUUID(); await client.topicSet(id, { hubUrl }); console.log('created fake topic', id); return id; } async function newSubscriber() { - const id = uuid.v4(); + const id = crypto.randomUUID(); await client.subscriberSetVerify(id); console.log('created fake subscriber', id); return id; diff --git a/test-e2e/test-one.js b/test-e2e/test-one.js index 8c6fdf2..e051983 100644 --- a/test-e2e/test-one.js +++ b/test-e2e/test-one.js @@ -1,7 +1,6 @@ 'use strict'; -const assert = require('assert'); -const uuid = require('uuid'); +const crypto = require('crypto'); const FakeServersClient = require('./fake-servers-client'); const subscriberPort = process.env.FAKE_SUBSCRIBER_PORT || 9876; @@ -14,14 +13,14 @@ const hubUrl = `http://${hubAddress}:${hubPort}/`; const client = new FakeServersClient(listenAddress, subscriberPort, topicPort); async function newTopic() { - const id = uuid.v4(); + const id = crypto.randomUUID(); await client.topicSet(id, { hubUrl }); console.log('created fake topic', id); return id; } async function newSubscriber() { - const id = uuid.v4(); + const id = crypto.randomUUID(); await client.subscriberSetVerify(id); console.log('created fake subscriber', id); return id; @@ -32,7 +31,7 @@ async function newSubscriber() { const subscriberId = await newSubscriber(); const result = await client.subscribe(hubUrl, subscriberId, topicId); - console.log('subscribed', { status: result.status, headers: result.headers, data: result.data }); + console.log('subscribed', { status: result.statusCode, headers: result.headers, body: result.body }); console.log('done'); })().catch((e) => { diff --git a/test/src/common.js b/test/src/common.js index a9b228c..e46f96a 100644 --- a/test/src/common.js +++ b/test/src/common.js @@ -23,47 +23,71 @@ describe('Common', function () { }); }); // freezeDeep - describe('axiosResponseLogData', function () { + describe('gotResponseLogData', function () { it('covers', function () { const response = { - status: 200, - statusText: 'OK', + statusCode: 200, + statusMessage: 'OK', headers: { 'Content-Type': 'text/plain', }, otherData: 'blah', - data: 'Old Mother West Wind had stopped to talk with the Slender Fir Tree. "I\'ve just come across the Green Meadows," said Old Mother West Wind, “and there I saw the Best Thing in the World.”', + body: 'Old Mother West Wind had stopped to talk with the Slender Fir Tree. "I\'ve just come across the Green Meadows," said Old Mother West Wind, “and there I saw the Best Thing in the World.”', + timings: { + phases: { + total: 87, + }, + }, + retryCount: 2, + redirectUrls: ['https://example.com/clip/Thornton_Burgess'], }; const expected = { - status: 200, - statusText: 'OK', + statusCode: 200, + statusMessage: 'OK', headers: { 'Content-Type': 'text/plain', }, - data: 'Old Mother West Wind had stopped to talk with the Slender Fir Tree. "I\'ve just come across the Green... (184 bytes)', + body: 'Old Mother West Wind had stopped to talk with the Slender Fir Tree. "I\'ve just come across the Green... (184 bytes)', + elapsedTimeMs: 87, + retryCount: 2, + redirectUrls: ['https://example.com/clip/Thornton_Burgess'], + }; + const result = common.gotResponseLogData(response); + assert.deepStrictEqual(result, expected); + }); + it('covers buffer data', function () { + const response = { + statusCode: 200, + statusMessage: 'OK', + body: Buffer.from('Old Mother West Wind had stopped to talk with the Slender Fir Tree. "I\'ve just come across the Green Meadows," said Old Mother West Wind, “and there I saw the Best Thing in the World.”'), + }; + const expected = { + statusCode: 200, + statusMessage: 'OK', + body: '', }; - const result = common.axiosResponseLogData(response); + const result = common.gotResponseLogData(response); assert.deepStrictEqual(result, expected); }); it('covers no data', function () { const response = { - status: 200, - statusText: 'OK', + statusCode: 200, + statusMessage: 'OK', headers: { 'Content-Type': 'text/plain', }, }; const expected = { - status: 200, - statusText: 'OK', + statusCode: 200, + statusMessage: 'OK', headers: { 'Content-Type': 'text/plain', }, }; - const result = common.axiosResponseLogData(response); + const result = common.gotResponseLogData(response); assert.deepStrictEqual(result, expected); }); - }); // axiosResponseLogData + }); // gotResponseLogData describe('topicLeaseDefaults', function () { it('supplies necessary properties', function () { diff --git a/test/src/communication.js b/test/src/communication.js index d5ebe54..8be9241 100644 --- a/test/src/communication.js +++ b/test/src/communication.js @@ -37,20 +37,23 @@ describe('Communication', function () { communication = new Communication(stubLogger, stubDb, options); }); - describe('Axios timing coverage', function () { - const request = {}; - const response = { - config: request, - }; - it('tags request', function () { - communication.axios.interceptors.request.handlers[0].fulfilled(request); - assert(request.startTimestampMs); + describe('_init', function () { + it('covers', async function () { + await communication._init(); + await communication._init(); + assert(communication.Got); + assert(communication.got); }); - it('tags response', function () { - communication.axios.interceptors.response.handlers[0].fulfilled(response); - assert(response.elapsedTimeMs); + }); + + describe('_onRetry', function () { + it('covers', function () { + const error = {}; + const retryCount = 1; + communication._onRetry(error, retryCount); + assert(communication.logger.debug.called); }); - }); // Axios timing coverage + }); describe('userAgentString', function () { it('has default behavior', function () { @@ -110,134 +113,8 @@ describe('Communication', function () { it('hashes', function () { const result = Communication.contentHash(content, algorithm); assert.strictEqual(result, expected); - }) - }); - - describe('Axios Configurations', function () { - let requestUrl, expectedUrl, topicUrl; - beforeEach(function () { - requestUrl = 'https://example.com/callback/?id=123'; - expectedUrl = 'https://example.com/callback/'; - topicUrl = 'http://example.com/blog/'; - }); - it('_axiosConfig', function () { - const method = 'GET'; - const contentType = 'text/plain'; - const body = undefined; - const params = { - 'extra_parameter': 'foobar', - }; - const expectedUrlObj = new URL('https://example.com/callback/?id=123&extra_parameter=foobar'); - const expected = { - method, - url: 'https://example.com/callback/', - headers: { - 'Content-Type': 'text/plain', - }, - params: expectedUrlObj.searchParams, - responseType: 'text', - }; - const result = Communication._axiosConfig(method, requestUrl, body, params, { - 'Content-Type': contentType, - }); - delete result.transformResponse; - assert.deepStrictEqual(result, expected); - }); - it('_axiosConfig covers defaults', function () { - const method = 'OPTIONS'; - const expectedUrlObj = new URL(requestUrl); - const expected = { - method, - url: expectedUrl, - headers: {}, - params: expectedUrlObj.searchParams, - responseType: 'text', - }; - const result = Communication._axiosConfig(method, requestUrl); - delete result.transformResponse; - assert.deepStrictEqual(result, expected); - }); - it('covers null response transform', function () { - const result = Communication._axiosConfig('GET', 'https://example.com/', undefined, {}, {}); - result.transformResponse[0](); - }); - it('_intentVerifyAxiosConfig', function () { - const mode = 'subscribe'; - const leaseSeconds = 864000; - const challenge = 'abcxyz'; - const expectedUrlObj = new URL(`${requestUrl}&hub.mode=${mode}&hub.topic=${encodeURIComponent(topicUrl)}&hub.challenge=${challenge}&hub.lease_seconds=${leaseSeconds}`); - const expected = { - method: 'GET', - url: expectedUrl, - headers: {}, - params: expectedUrlObj.searchParams, - responseType: 'text', - }; - const result = Communication._intentVerifyAxiosConfig(requestUrl, topicUrl, mode, leaseSeconds, challenge); - delete result.transformResponse; - assert.deepStrictEqual(result, expected); - }); - it('_intentDenyAxiosConfig', function () { - const reason = 'something'; - const expectedUrlObj = new URL(`${requestUrl}&hub.mode=denied&hub.topic=${encodeURIComponent(topicUrl)}&hub.reason=${reason}`); - const expected = { - method: 'GET', - url: expectedUrl, - headers: {}, - params: expectedUrlObj.searchParams, - responseType: 'text', - }; - const result = Communication._intentDenyAxiosConfig(requestUrl, topicUrl, reason); - delete result.transformResponse; - assert.deepStrictEqual(result, expected); - }); - it('_publisherValidationAxiosConfig', function () { - const topic = { - url: topicUrl, - publisherValidationUrl: 'https://example.com/publisher/', - }; - const verification = { - callback: requestUrl, - topic: topicUrl, - }; - const expectedUrlObj = new URL(topic.publisherValidationUrl); - const expected = { - method: 'POST', - url: topic.publisherValidationUrl, - data: { - callback: requestUrl, - topic: topicUrl, - }, - headers: { - 'Content-Type': 'application/json', - }, - params: expectedUrlObj.searchParams, - responseType: 'text', - }; - const result = Communication._publisherValidationAxiosConfig(topic, verification); - delete result.transformResponse; - assert.deepStrictEqual(result, expected); - }); - it('_topicFetchAxiosConfig', function () { - const topic = { - url: topicUrl, - contentType: 'text/plain', - }; - const expectedUrlObj = new URL(topicUrl); - const expected = { - method: 'GET', - url: topicUrl, - params: expectedUrlObj.searchParams, - headers: { - Accept: 'text/plain, */*;q=0.9', - }, - responseType: 'text', - }; - const result = Communication._topicFetchAxiosConfig(topic); - delete result.transformResponse; - assert.deepStrictEqual(result, expected); }); - }); // Axios Configurations + }); describe('verificationProcess', function () { const challenge = 'a_challenge'; @@ -263,13 +140,13 @@ describe('Communication', function () { sinon.stub(Communication, 'generateChallenge').resolves(challenge); sinon.stub(communication, 'publisherValidate').resolves(true); - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'text/plain', }, - data: challenge, + body: challenge, }); communication.db.verificationGetById.resolves(verification); @@ -312,7 +189,7 @@ describe('Communication', function () { await communication.verificationProcess(dbCtx, callback, topicId, requestId); assert(communication.db.verificationRelease.called); - assert(!communication.axios.called); + assert(!communication.got.called); }); it('denies subscription to deleted topic', async function () { @@ -357,8 +234,8 @@ describe('Communication', function () { }); it('handles request error', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').throws(new Error()); + communication.got.restore(); + sinon.stub(communication, 'got').rejects(new Error()); await communication.verificationProcess(dbCtx, callback, topicId, requestId); @@ -366,9 +243,9 @@ describe('Communication', function () { }); it('handles 500 response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 500, + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 500, }); await communication.verificationProcess(dbCtx, callback, topicId, requestId); @@ -377,9 +254,9 @@ describe('Communication', function () { }); it('handles non-200 response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 400, + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 400, }); await communication.verificationProcess(dbCtx, callback, topicId, requestId); @@ -426,14 +303,14 @@ describe('Communication', function () { communication.db.verificationGetById.restore(); verification.mode = 'unsubscribe'; sinon.stub(communication.db, 'verificationGetById').resolves(verification); - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'text/plain', }, - data: 'not the challenge', + body: 'not the challenge', }); await communication.verificationProcess(dbCtx, callback, topicId, requestId); @@ -470,9 +347,9 @@ describe('Communication', function () { httpRemoteAddr: '127.0.0.0', }; - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'application/json', }, @@ -491,10 +368,10 @@ describe('Communication', function () { }); it('succeeds with rejection', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 400, - statusText: 'Bad Request', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 400, + statusMessage: 'Bad Request', headers: { 'content-type': 'application/json', }, @@ -508,10 +385,10 @@ describe('Communication', function () { }); it('defers on request server error', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 502, - statusText: 'Bad Gateway', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 502, + statusMessage: 'Bad Gateway', headers: { 'content-type': 'text/plain', }, @@ -523,8 +400,8 @@ describe('Communication', function () { }); it('handles request error', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').throws(new Error()); + communication.got.restore(); + sinon.stub(communication, 'got').rejects(new Error()); const result = await communication.publisherValidate(dbCtx, topic, verification); @@ -546,16 +423,16 @@ describe('Communication', function () { requestId = '7d37ea20-4ef7-417e-a08d-c0ba71269ab1'; topicId = '234ec6fb-f1cd-4ac3-8ea9-29ed42ae0e21'; - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'text/plain', link: '; rel="hub"', 'last-modified': 'Thu, 18 Nov 2021 20:34:35 GMT', 'etag': '"9c104-1673e-5d1161636d742"', }, - data: 'Jackdaws love my big sphinx of quartz.', + body: 'Jackdaws love my big sphinx of quartz.', }); communication.db.topicGetById.resolves(topic); @@ -580,12 +457,12 @@ describe('Communication', function () { await communication.topicFetchProcess(dbCtx, topicId, requestId); - assert(!communication.axios.called); + assert(!communication.got.called); }); it('handles request error', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').throws(new Error()); + communication.got.restore(); + sinon.stub(communication, 'got').rejects(new Error()); await communication.topicFetchProcess(dbCtx, topicId, requestId); @@ -593,10 +470,10 @@ describe('Communication', function () { }); it('handles 500 response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 500, - statusText: 'Internal Server Error', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 500, + statusMessage: 'Internal Server Error', headers: { 'content-type': 'text/plain', }, @@ -608,10 +485,10 @@ describe('Communication', function () { }); it('handles bad response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 404, - statusText: 'Not Found', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 404, + statusMessage: 'Not Found', headers: { 'content-type': 'text/plain', }, @@ -637,8 +514,8 @@ describe('Communication', function () { topic.httpLastModified = 'Thu, 18 Nov 2021 20:34:35 GMT'; topic.httpEtag = '"9c104-1673e-5d1161636d742"'; communication.db.topicGetById.resolves(topic); - communication.axios.resolves({ - status: 304, + communication.got.resolves({ + statusCode: 304, }); await communication.topicFetchProcess(dbCtx, topicId, requestId); @@ -655,15 +532,15 @@ describe('Communication', function () { }); it('updates content with lax link enforcement', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'text/plain', link: '; rel="hub"', }, - data: 'Jackdaws love my big sphinx of quartz.', + body: 'Jackdaws love my big sphinx of quartz.', }); communication.options.communication.strictTopicHubLink = false; @@ -675,15 +552,15 @@ describe('Communication', function () { }); it('deletes topic when hub relation unsatisfied', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'text/plain', link: '; rel="hub"', }, - data: 'Jackdaws love my big sphinx of quartz.', + body: 'Jackdaws love my big sphinx of quartz.', }); await communication.topicFetchProcess(dbCtx, topicId, requestId); @@ -714,13 +591,13 @@ describe('Communication', function () { signatureAlgorithm: 'sha512', }; - sinon.stub(communication, 'axios').resolves({ - status: 200, - statusText: 'OK', + sinon.stub(communication, 'got').resolves({ + statusCode: 200, + statusMessage: 'OK', headers: { 'content-type': 'text/plain', }, - data: 'Jackdaws love my big sphinx of quartz.', + body: 'Jackdaws love my big sphinx of quartz.', }); communication.db.topicGetContentById.resolves(topic); @@ -756,8 +633,8 @@ describe('Communication', function () { }); it('handles request error', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').throws(); + communication.got.restore(); + sinon.stub(communication, 'got').throws(); await communication.subscriptionDeliveryProcess(dbCtx, subscriptionId, requestId); @@ -765,10 +642,10 @@ describe('Communication', function () { }); it('handles 5xx response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 500, - statusText: 'Internal Server Error', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 500, + statusMessage: 'Internal Server Error', headers: { 'content-type': 'text/plain', }, @@ -780,10 +657,10 @@ describe('Communication', function () { }); it('handles 4xx response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 404, - statusText: 'Not Found', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 404, + statusMessage: 'Not Found', headers: { 'content-type': 'text/plain', }, @@ -795,10 +672,10 @@ describe('Communication', function () { }); it('handles 410 response', async function () { - communication.axios.restore(); - sinon.stub(communication, 'axios').resolves({ - status: 410, - statusText: 'Gone', + communication.got.restore(); + sinon.stub(communication, 'got').resolves({ + statusCode: 410, + statusMessage: 'Gone', headers: { 'content-type': 'text/plain', }, -- 2.45.2