X-Git-Url: https://git.squeep.com/?a=blobdiff_plain;f=lib%2Fcommunication.js;h=18a6e588070e39bb49790433885189703a3bc5d0;hb=cc52f66ba8522b6bc7002dfba79c1162a51aef0e;hp=540eaafa14623ca251a2a7b8d1267bf26a57105b;hpb=1c4bb7e3bbdc1121ceba373c5be4459521197155;p=squeep-indieauth-helper diff --git a/lib/communication.js b/lib/communication.js index 540eaaf..18a6e58 100644 --- a/lib/communication.js +++ b/lib/communication.js @@ -1,24 +1,30 @@ 'use strict'; -const axios = require('axios'); const { mf2 } = require('microformats-parser'); -const { base64ToBase64URL } = require('@squeep/base64url'); const { parse: parseLinkHeader } = require('@squeep/web-linking'); const { Iconv } = require('iconv'); const { version: packageVersion, name: packageName } = require('../package.json'); -const { performance } = require('perf_hooks'); const { randomBytes, createHash } = require('crypto'); const { promisify } = require('util'); const randomBytesAsync = promisify(randomBytes); +const { Address4, Address6 } = require('ip-address'); +const dns = require('dns'); const common = require('./common'); const Enum = require('./enum'); +const { ValidationError } = require('./errors'); const _fileScope = common.fileScope(__filename); +const noDotPathRE = /(\/\.\/|\/\.\.\/)/; +const v6HostRE = /\[[0-9a-f:]+\]/; +const loopback4 = new Address4('127.0.0.0/8'); +const scopeSplitRE = / +/; + class Communication { /** * @param {Console} logger * @param {Object} options + * @param {Number=} options.timeout * @param {Object=} options.userAgent * @param {String=} options.userAgent.product * @param {String=} options.userAgent.version @@ -27,33 +33,77 @@ class Communication { constructor(logger, options = {}) { this.logger = logger; this.options = options; - this.axios = axios.create({ - headers: { - [Enum.Header.UserAgent]: Communication._userAgentString(options.userAgent), - [Enum.Header.Accept]: 'text/html, text/*;q=0.9, application/xhtml+xml;q=0.8, application/xml;q=0.8, */*;q=0.1', - }, - }); - this.axios.interceptors.request.use((request) => { - request.startTimestampMs = performance.now(); - return request; - }); - this.axios.interceptors.response.use((response) => { - response.elapsedTimeMs = performance.now() - response.config.startTimestampMs; - return response; - }); + + this._defaultAccept = options?.defaultAccept || 'text/html, text/*;q=0.9, application/xhtml+xml;q=0.8, application/xml;q=0.8, */*;q=0.1'; + this._jsonAccept = options?.jsonAccept || [Enum.ContentType.ApplicationJson, Enum.ContentType.Any + ';q=0.1'].join(', '); + + this.Got = undefined; + this.got = this._init; // Do the dynamic import on first attempt to use client. + } + + + /** + * Do a little dance to support this ESM client. + */ + async _init(...args) { + if (!this.Got) { + // For some reason eslint is confused about import being supported here. + // eslint-disable-next-line + this.Got = await import('got'); + this.got = this.Got.got.extend({ + headers: { + [Enum.Header.UserAgent]: Communication._userAgentString(this.options.userAgent), + [Enum.Header.Accept]: this._defaultAccept, + }, + timeout: { + request: this.options.timeout || 120000, + }, + hooks: { + beforeRetry: [ + this._onRetry, + ], + }, + }); + } + if (args.length) { + return this.got(...args); + } + } + + + /** + * Take notes on transient retries. + * @param {*} error + * @param {*} retryCount + */ + _onRetry(error, retryCount) { + const _scope = _fileScope('_onRetry'); + this.logger.debug(_scope, 'retry', { retryCount, error }); } + /** + * Encode hashed verifier data for PKCE. + * @param {BinaryLike} verifier + * @returns {String} + */ static _challengeFromVerifier(verifier) { const hash = createHash('sha256'); hash.update(verifier); - return base64ToBase64URL(hash.digest('base64')); + return hash.digest('base64url'); } + + /** + * @typedef PKCEData + * @property {String} codeChallengeMethod + * @property {String} codeVerifier + * @property {String} codeChallenge + */ /** * Create a code verifier and its challenge. - * @param {Number} length - * @returns {Object} + * @param {Number} length of verifier string, between 43 and 128 + * @returns {Promise} */ static async generatePKCE(length = 128) { if (length < 43 || length > 128) { @@ -62,7 +112,7 @@ class Communication { const bufferLength = Math.floor(length * 3 / 4); const randomBuffer = await randomBytesAsync(bufferLength); - const verifier = base64ToBase64URL(randomBuffer.toString('base64')); + const verifier = randomBuffer.toString('base64url'); const challenge = Communication._challengeFromVerifier(verifier); @@ -116,32 +166,6 @@ class Communication { } - /** - * A request config skeleton. - * @param {String} method - * @param {URL} urlObj - * @param {String=} body - * @param {Object=} params - * @param {Object=} headers - * @returns {Object} - */ - static _axiosConfig(method, urlObj, body, params = {}, headers = {}) { - const config = { - method, - url: `${urlObj.origin}${urlObj.pathname}`, - params: urlObj.searchParams, - headers, - ...(body && { data: body }), - // Setting this does not appear to be enough to keep axios from parsing JSON response into object - responseType: 'text', - // So force the matter by eliding all response transformations - transformResponse: [ (res) => res ], - }; - Object.entries(params).map(([k, v]) => config.params.set(k, v)); - return config; - } - - /** * Isolate the base of a url. * mf2 parser needs this so that relative links can be made absolute. @@ -175,7 +199,7 @@ class Communication { mediaType: mediaType.toLowerCase() || defaultContentType, params: params.reduce((obj, param) => { const [field, value] = param.split('='); - const isQuoted = value && value.charAt(0) === '"' && value.charAt(value.length - 1) === '"'; + const isQuoted = value?.startsWith('"') && value?.endsWith('"'); obj[field.toLowerCase()] = isQuoted ? value.slice(1, value.length - 1) : value; return obj; }, {}), @@ -184,9 +208,10 @@ class Communication { /** - * Parse and add any header link relations to mf data. + * Parse and add any header link relations from response to microformat data. * @param {Object} microformat * @param {Object} response + * @param {Object} response.headers */ _mergeLinkHeader(microformat, response) { const _scope = _fileScope('_mergeLinkHeader'); @@ -239,7 +264,7 @@ class Communication { * Retrieve and parse microformat data from url. * N.B. this absorbs any errors! * @param {URL} urlObj - * @returns {Object} + * @returns {Promise} */ async fetchMicroformat(urlObj) { const _scope = _fileScope('fetchMicroformat'); @@ -250,27 +275,34 @@ class Communication { }; let response; try { - const fetchMicroformatConfig = Communication._axiosConfig('GET', urlObj); - response = await this.axios(fetchMicroformatConfig); + const fetchMicroformatConfig = { + method: 'GET', + url: urlObj, + responseType: 'buffer', + }; + response = await this.got(fetchMicroformatConfig); } catch (e) { this.logger.error(_scope, 'microformat request failed', { error: e, ...logInfoData }); return; } - logInfoData.response = common.axiosResponseLogData(response); + logInfoData.response = common.gotResponseLogData(response); // Normalize to utf8. - let body = response.data; + let body; const contentType = Communication._parseContentType(response.headers[Enum.Header.ContentType.toLowerCase()]); + // If a charset was specified, and it's not utf8ish, attempt to transliterate it to utf8. const nonUTF8Charset = !/utf-*8/i.test(contentType.params.charset) && contentType.params.charset; if (nonUTF8Charset) { try { const iconv = new Iconv(nonUTF8Charset, 'utf-8//translit//ignore'); - body = iconv.convert(body).toString('utf8'); + body = iconv.convert(response.body).toString('utf8'); } catch (e) { // istanbul ignore next this.logger.error(_scope, 'iconv conversion error', { error: e, ...logInfoData }); // Try to carry on, maybe the encoding will work anyhow... } + } else { + body = response.body.toString('utf8'); } let microformat = {}; @@ -296,7 +328,7 @@ class Communication { * Retrieve and parse JSON. * N.B. this absorbs any errors! * @param {URL} urlObj - * @returns {Object} + * @returns {Promise} */ async fetchJSON(urlObj) { const _scope = _fileScope('fetchJSON'); @@ -306,24 +338,230 @@ class Communication { }; let response; try { - const fetchJSONConfig = Communication._axiosConfig('GET', urlObj, undefined, undefined, { - [Enum.Header.Accept]: [Enum.ContentType.ApplicationJson, Enum.ContentType.Any + ';q=0.1'].join(', '), - }); - response = await this.axios(fetchJSONConfig); + const fetchJSONConfig = { + method: 'GET', + url: urlObj, + headers: { + [Enum.Header.Accept]: this._jsonAccept, + }, + responseType: 'json', + }; + response = await this.got(fetchJSONConfig); } catch (e) { this.logger.error(_scope, 'json request failed', { error: e, ...logInfoData }); return; } - logInfoData.response = common.axiosResponseLogData(response); + logInfoData.response = common.gotResponseLogData(response); + + return response.body; + } + + + /** + * Validate a url has a specific schema. + * @param {URL} urlObj + * @param {String[]} validSchemes + */ + static _urlValidScheme(urlObj, validSchemes = ['http:', 'https:']) { + if (!validSchemes.includes(urlObj.protocol)) { + throw new ValidationError(`unsupported url scheme '${urlObj.protocol}'`); + } + } + + + /** + * Validate a url does not include some components. + * @param {URL} urlObj + * @param {String[]} disallowed + */ + static _urlPartsDisallowed(urlObj, disallowed) { + disallowed.forEach((part) => { + if (urlObj[part]) { // eslint-disable-line security/detect-object-injection + throw new ValidationError(`url cannot contain ${common.properURLComponentName(part)}`); + } + }); + } + + + /** + * Validate a url does not have relative path. + * @param {String} url + */ + static _urlPathNoDots(url) { + if (noDotPathRE.test(url)) { + throw new ValidationError('relative path segment not valid'); + } + } + + + /** + * Validate a url does not have a hostname which is an ip address. + * N.B. Sets isLoopback on urlObj + * @param {URL} urlObj + * @param {Boolean} allowLoopback + * @returns {Promise} + */ + static async _urlNamedHost(urlObj, allowLoopback, resolveHostname) { + let address; + if (v6HostRE.test(urlObj.hostname)) { + /** + * We do not need to worry about the Address6() failing to parse, + * as if it looks like an ipv6 addr but is not valid, the URL() + * call would already have failed. + */ + address = new Address6(urlObj.hostname.slice(1, urlObj.hostname.length - 1)); + /* succeeded parsing as ipv6, reject unless loopback */ + urlObj.isLoopback = address.isLoopback(); + } else { + try { + address = new Address4(urlObj.hostname); + /* succeeded parsing as ipv4, reject unless loopback */ + urlObj.isLoopback = address.isInSubnet(loopback4); + } catch (e) { + /* did not parse as ip, carry on */ + } + } + + if (resolveHostname && !urlObj.isLoopback) { + /** + * Resolve hostname to check for localhost. + * This is more complicated due to SSRF mitigation: + * If the hostname does not end with a ., we also resolve that, + * and complain if the two resolutions do not match, assuming + * malicious intent for the server to resolve a local record. + */ + const hostnames = [urlObj.hostname]; + if (!urlObj.hostname.endsWith('.')) { + hostnames.push(urlObj.hostname + '.'); + } + const settledResolutions = await Promise.allSettled(hostnames.map((hostname) => dns.promises.lookup(hostname, { + all: true, + verbatim: true, + }))); + // If any resolution failed, bail. + if (settledResolutions + .map((resolution) => resolution.status) + .includes('rejected')) { + throw new ValidationError('could not resolve hostname'); + } + + // extract each resolution value, array of {address,family} + const resolutions = settledResolutions.map((resolution) => resolution.value); + + // If there were two resolutions, ensure they returned identical results. + if (resolutions.length > 1) { + // create set of addresses for each resolution + const addressSets = resolutions.map((addrs) => { + return new Set((addrs || []).map((a) => a.address)); + }); + const differences = common.setSymmetricDifference(...addressSets); + if (differences.size) { + throw new ValidationError('inconsistent hostname resolution'); + } + } + const resolvedHost = resolutions[0] || []; + + // Persist the loopback state + urlObj.isLoopback = resolvedHost.reduce((acc, resolved) => { + let addr; + switch (resolved.family) { + case 4: + addr = new Address4(resolved.address); + return acc || addr.isInSubnet(loopback4); + case 6: + addr = new Address6(resolved.address); + return acc || addr.isLoopback(); + default: + return acc; + } + }, false); + } + + if (address + && (!urlObj.isLoopback || !allowLoopback)) { + throw new ValidationError('hostname cannot be IP'); + } + } + + + /** + * Ensure a url meets the requirements to be a profile uri. + * @param {String} url + * @param {Object} validationOptions + * @param {Boolean} validationOptions.allowLoopback + * @param {Boolean} validationOptions.resolveHostname + * @returns {Promise} + */ + async validateProfile(url, validationOptions) { + const _scope = _fileScope('validateProfile'); + const errorScope = 'invalid profile url'; + + const options = Object.assign({ + allowLoopback: false, + resolveHostname: false, + }, validationOptions); + + let profile; + try { + profile = new URL(url); + } catch (e) { + this.logger.debug(_scope, 'failed to parse url', { error: e, url }); + throw new ValidationError(`${errorScope}: unparsable`); + } + profile.isLoopback = false; + + try { + Communication._urlValidScheme(profile); + Communication._urlPartsDisallowed(profile, ['hash', 'username', 'password', 'port']); + Communication._urlPathNoDots(url); + await Communication._urlNamedHost(profile, options.allowLoopback, options.resolveHostname); + } catch (e) { + this.logger.debug(_scope, 'profile url not valid', { url, error: e }); + throw new ValidationError(`${errorScope}: ${e.message}`); + } + + return profile; + } + + + /** + * Ensure a url meets the requirements to be a client identifier. + * Sets 'isLoopback' on returned URL object to true if hostname is or resolves to a loopback ip. + * @param {String} url + * @param {Object} validationOptions + * @param {Boolean} validationOptions.allowLoopback + * @param {Boolean} validationOptions.resolveHostname + * @returns {Promise} + */ + async validateClientIdentifier(url, validationOptions) { + const _scope = _fileScope('validateClientIdentifier'); + const errorScope = 'invalid client identifier url'; + + const options = Object.assign({ + allowLoopback: true, + resolveHostname: true, + }, validationOptions); + + let clientId; + try { + clientId = new URL(url); + } catch (e) { + this.logger.debug(_scope, 'failed to parse url', { error: e, url }); + throw new ValidationError('invalid client identifier url: unparsable'); + } + clientId.isLoopback = false; - let data; try { - data = JSON.parse(response.data); + Communication._urlValidScheme(clientId); + Communication._urlPartsDisallowed(clientId, ['hash', 'username', 'password']); + Communication._urlPathNoDots(url); + await Communication._urlNamedHost(clientId, options.allowLoopback, options.resolveHostname); } catch (e) { - this.logger.error(_scope, 'json parsing failed', { error: e, ...logInfoData }); + this.logger.debug(_scope, 'client identifier url not valid', { url, error: e }); + throw new ValidationError(`${errorScope}: ${e.message}`); } - return data; + return clientId; } @@ -334,10 +572,24 @@ class Communication { */ /** * Retrieve and parse client identifier endpoint data. + * N.B. Assumes urlObj has passed validateClientIdentifier. * @param {URL} urlObj * @returns {ClientIdentifierData|undefined} mf2 data filtered for h-app items, or undefined if url could not be fetched */ async fetchClientIdentifier(urlObj) { + const _scope = _fileScope('fetchClientIdentifier'); + + // Loopback address will eschew client fetch, return empty data. + const isLoopbackResult = { + rels: {}, + items: [], + }; + + // Set by validation method in case of loopback ip hostname + if (urlObj.isLoopback) { + return isLoopbackResult; + } + const mfData = await this.fetchMicroformat(urlObj); if (!mfData) { return undefined; @@ -350,7 +602,7 @@ class Communication { let urlMatched = false; const itemType = item.type || []; if ((itemType.includes('h-app') || itemType.includes('h-x-app')) - && (item.properties && item.properties.url)) { + && (item?.properties?.url)) { item.properties.url.forEach((url) => { try { const hUrl = new URL(url); @@ -379,6 +631,7 @@ class Communication { * @property {String} metadata.issuer * @property {String} metadata.authorizationEndpoint * @property {String} metadata.tokenEndpoint + * @property {String} metadata.ticketEndpoint * @property {String} metadata.introspectionEndpoint * @property {String} metadata.introspectionEndpointAuthMethodsSupported * @property {String} metadata.revocationEndpoint @@ -413,7 +666,7 @@ class Communication { // and populate profile fields with first-encountered card values. if (mfData && 'items' in mfData) { const hCards = mfData.items.filter((item) => - item.type && item.type.includes('h-card') && + item?.type?.includes('h-card') && item.properties && item.properties.url && item.properties.url.includes(urlObj.href)); hCards.forEach((hCard) => { Object.keys(profile).forEach((key) => { @@ -429,6 +682,7 @@ class Communication { Object.entries({ authorizationEndpoint: 'authorization_endpoint', // backwards compatibility tokenEndpoint: 'token_endpoint', // backwards compatibility + ticketEndpoint: 'ticket_endpoint', // backwards compatibility }).forEach(([p, r]) => { if (mfData && r in mfData.rels) { profile.metadata[p] = profile[p] = mfData.rels[r][0]; // eslint-disable-line security/detect-object-injection @@ -457,6 +711,7 @@ class Communication { issuer: 'issuer', authorizationEndpoint: 'authorization_endpoint', tokenEndpoint: 'token_endpoint', + ticketEndpoint: 'ticket_endpoint', introspectionEndpoint: 'introspection_endpoint', introspectionEndpointAuthMethodsSupported: 'introspection_endpoint_auth_methods_supported', revocationEndpoint: 'revocation_endpoint', @@ -475,7 +730,7 @@ class Communication { }); // Populate legacy profile fields. - ['authorizationEndpoint', 'tokenEndpoint'].forEach((f) => { + ['authorizationEndpoint', 'tokenEndpoint', 'ticketEndpoint'].forEach((f) => { if (f in profile.metadata) { profile[f] = profile.metadata[f]; // eslint-disable-line security/detect-object-injection } @@ -489,7 +744,8 @@ class Communication { /** - * POST to the auth endpoint, to redeem a code for a profile object. + * POST to the auth endpoint, to redeem a code for a profile or token. + * N.B. this absorbs any errors! * @param {URL} urlObj * @param {String} code * @param {String} codeVerifier @@ -497,37 +753,138 @@ class Communication { * @param {String} redirectURI * @returns {Object} */ + async redeemCode(urlObj, code, codeVerifier, clientId, redirectURI) { + const _scope = _fileScope('redeemCode'); + + const postRedeemCodeConfig = { + url: urlObj, + method: 'POST', + headers: { + [Enum.Header.Accept]: this._jsonAccept, + }, + form: { + 'grant_type': 'authorization_code', + code, + 'client_id': clientId, + 'redirect_uri': redirectURI, + 'code_verifier': codeVerifier, + }, + responseType: 'json', + }; + + try { + const response = await this.got(postRedeemCodeConfig); + return response.body; + } catch (e) { + this.logger.error(_scope, 'redeem code request failed', { error: e, url: urlObj.href }); + return; + } + } + + + /** + * Deprecated method name alias. + * @see redeemCode + * @param {URL} urlObj + * @param {String} code + * @param {Strin} codeVerifier + * @param {String} clientId + * @param {String} redirectURI + * @returns {Object} + */ async redeemProfileCode(urlObj, code, codeVerifier, clientId, redirectURI) { - const _scope = _fileScope('redeemProfileCode'); + return this.redeemCode(urlObj, code, codeVerifier, clientId, redirectURI); + } - const data = new URLSearchParams(); - Object.entries({ - 'grant_type': 'authorization_code', - code, - 'client_id': clientId, - 'redirect_uri': redirectURI, - 'code_verifier': codeVerifier, - }).forEach(([name, value]) => data.set(name, value)); - - const postRedeemProfileCodeConfig = Communication._axiosConfig('POST', urlObj, data.toString(), {}, { - [Enum.Header.ContentType]: Enum.ContentType.ApplicationForm, - [Enum.Header.Accept]: `${Enum.ContentType.ApplicationJson}, ${Enum.ContentType.Any};q=0.1`, - }); + + /** + * Verify a token with an IdP endpoint, using the Authorization header supplied. + * @param {URL} introspectionUrlObj + * @param {String} authorizationHeader + * @param {String} token + */ + async introspectToken(introspectionUrlObj, authorizationHeader, token) { + const _scope = _fileScope('introspectToken'); + + const postIntrospectConfig = { + url: introspectionUrlObj, + method: 'POST', + headers: { + [Enum.Header.Authorization]: authorizationHeader, + [Enum.Header.Accept]: this._jsonAccept, + }, + form: { + token, + }, + responseType: 'json', + }; try { - const response = await this.axios(postRedeemProfileCodeConfig); + const response = await this.got(postIntrospectConfig); + this.logger.debug(_scope, 'response', { response }); + // check status try { - return JSON.parse(response.data); + const { + active, + me, + client_id: clientId, + scope, + exp, + iat, + } = response.body; + + if (![true, false].includes(active)) { + throw new RangeError('missing required response field "active"'); + } + + return { + active, + ...(me && { me }), + ...(clientId && { clientId }), + ...(scope && { scope: scope.split(scopeSplitRE) }), + ...(exp && { exp: Number(exp) }), + ...(iat && { iat: Number(iat) }), + }; } catch (e) { - this.logger.error(_scope, 'failed to parse json', { error: e, response }); + this.logger.error(_scope, 'failed to parse json', { error: e, response: common.gotResponseLogData(response) }); throw e; } } catch (e) { - this.logger.error(_scope, 'redeem profile code request failed', { error: e, url: urlObj.href }); - return; + this.logger.error(_scope, 'introspect token request failed', { error: e, url: introspectionUrlObj.href }); + throw e; + } + } + + + /** + * Attempt to deliver a ticket to an endpoint. + * N.B. does not absorb errors + * @param {URL} ticketEndpointUrlObj + * @param {URL} resourceUrlObj + * @param {URL} subjectUrlObj + * @param {String} ticket + * @returns {Promise} + */ + async deliverTicket(ticketEndpointUrlObj, resourceUrlObj, subjectUrlObj, ticket) { + const _scope = _fileScope('deliverTicket'); + + try { + const ticketConfig = { + method: 'POST', + url: ticketEndpointUrlObj, + form: { + ticket, + resource: resourceUrlObj.href, + subject: subjectUrlObj.href, + }, + }; + return await this.got(ticketConfig); + } catch (e) { + this.logger.error(_scope, 'ticket delivery request failed', { error: e, url: ticketEndpointUrlObj.href }); + throw e; } } } -module.exports = Communication; \ No newline at end of file +module.exports = Communication;