3 const axios
= require('axios');
4 const { mf2
} = require('microformats-parser');
5 const { base64ToBase64URL
} = require('@squeep/base64url');
6 const { parse: parseLinkHeader
} = require('@squeep/web-linking');
7 const { Iconv
} = require('iconv');
8 const { version: packageVersion
, name: packageName
} = require('../package.json');
9 const { performance
} = require('perf_hooks');
10 const { randomBytes
, createHash
} = require('crypto');
11 const { promisify
} = require('util');
12 const randomBytesAsync
= promisify(randomBytes
);
13 const { Address4
, Address6
} = require('ip-address');
14 const dns
= require('dns');
15 dns
.lookupAsync
= dns
.lookupAsync
|| promisify(dns
.lookup
);
16 const common
= require('./common');
17 const Enum
= require('./enum');
18 const { ValidationError
} = require('./errors');
20 const _fileScope
= common
.fileScope(__filename
);
22 const noDotPathRE
= /(\/\.\/|\/\.\.\/)/;
23 const v6HostRE
= /\[[0-9a-f:]+\]/;
24 const loopback4
= new Address4('127.0.0.0/8');
28 * @param {Console} logger
29 * @param {Object} options
30 * @param {Object=} options.userAgent
31 * @param {String=} options.userAgent.product
32 * @param {String=} options.userAgent.version
33 * @param {String=} options.userAgent.implementation
35 constructor(logger
, options
= {}) {
37 this.options
= options
;
38 this.axios
= axios
.create({
40 [Enum
.Header
.UserAgent
]: Communication
._userAgentString(options
.userAgent
),
41 [Enum
.Header
.Accept
]: 'text/html, text/*;q=0.9, application/xhtml+xml;q=0.8, application/xml;q=0.8, */*;q=0.1',
44 this.axios
.interceptors
.request
.use((request
) => {
45 request
.startTimestampMs
= performance
.now();
48 this.axios
.interceptors
.response
.use((response
) => {
49 response
.elapsedTimeMs
= performance
.now() - response
.config
.startTimestampMs
;
55 static _challengeFromVerifier(verifier
) {
56 const hash
= createHash('sha256');
57 hash
.update(verifier
);
58 return base64ToBase64URL(hash
.digest('base64'));
62 * Create a code verifier and its challenge.
63 * @param {Number} length
66 static async
generatePKCE(length
= 128) {
67 if (length
< 43 || length
> 128) {
68 throw new RangeError('InvalidLength');
71 const bufferLength
= Math
.floor(length
* 3 / 4);
72 const randomBuffer
= await
randomBytesAsync(bufferLength
);
73 const verifier
= base64ToBase64URL(randomBuffer
.toString('base64'));
75 const challenge
= Communication
._challengeFromVerifier(verifier
);
78 codeChallengeMethod: 'S256',
79 codeVerifier: verifier
,
80 codeChallenge: challenge
,
86 * Check a challenge with a verifier.
87 * @param {String} codeChallenge
88 * @param {String} codeVerifier
89 * @param {String} codeChallengeMethod
92 static verifyChallenge(codeChallenge
, codeVerifier
, codeChallengeMethod
) {
93 switch (codeChallengeMethod
) {
96 const challenge
= Communication
._challengeFromVerifier(codeVerifier
);
97 return challenge
=== codeChallenge
;
101 throw new Error('unsupported challenge method');
107 * Assemble a suitable User-Agent value.
108 * @param {Object} userAgentConfig
109 * @param {String=} userAgentConfig.product
110 * @param {String=} userAgentConfig.version
111 * @param {String=} userAgentConfig.implementation
114 static _userAgentString(userAgentConfig
) {
115 // eslint-disable-next-line security/detect-object-injection
116 const _conf
= (field
, def
) => (userAgentConfig
&& field
in userAgentConfig
) ? userAgentConfig
[field
] : def
;
117 const product
= _conf('product', packageName
).split('/').pop();
118 const version
= _conf('version', packageVersion
);
119 let implementation
= _conf('implementation', Enum
.Specification
);
120 if (implementation
) {
121 implementation
= ` (${implementation})`;
123 return `${product}/${version}${implementation}`;
128 * A request config skeleton.
129 * @param {String} method
130 * @param {URL} urlObj
131 * @param {String=} body
132 * @param {Object=} params
133 * @param {Object=} headers
136 static _axiosConfig(method
, urlObj
, body
, params
= {}, headers
= {}) {
139 url: `${urlObj.origin}${urlObj.pathname}`,
140 params: urlObj
.searchParams
,
142 ...(body
&& { data: body
}),
143 // Setting this does not appear to be enough to keep axios from parsing JSON response into object
144 responseType: 'text',
145 // So force the matter by eliding all response transformations
146 transformResponse: [ (res
) => res
],
148 Object
.entries(params
).map(([k
, v
]) => config
.params
.set(k
, v
));
154 * Isolate the base of a url.
155 * mf2 parser needs this so that relative links can be made absolute.
156 * @param {URL} urlObj
159 static _baseUrlString(urlObj
) {
160 const baseUrl
= new URL(urlObj
);
161 const lastSlashIdx
= baseUrl
.pathname
.lastIndexOf('/');
162 if (lastSlashIdx
> 0) {
163 baseUrl
.pathname
= baseUrl
.pathname
.slice(0, lastSlashIdx
+ 1);
170 * Convert a Content-Type string to normalized components.
172 * N.B. this ill-named non-parsing implementation will not work
173 * if a parameter value for some reason includes a ; or = within
175 * @param {String} contentTypeHeader
176 * @returns {Object} contentType
177 * @returns {String} contentType.mediaType
178 * @returns {Object} contentType.params
180 static _parseContentType(contentTypeHeader
, defaultContentType
= Enum
.ContentType
.ApplicationOctetStream
) {
181 const [ mediaType
, ...params
] = (contentTypeHeader
|| '').split(/ *; */
);
183 mediaType: mediaType
.toLowerCase() || defaultContentType
,
184 params: params
.reduce((obj
, param
) => {
185 const [field
, value
] = param
.split('=');
186 const isQuoted
= value
&& value
.charAt(0) === '"' && value
.charAt(value
.length
- 1) === '"';
187 obj
[field
.toLowerCase()] = isQuoted
? value
.slice(1, value
.length
- 1) : value
;
195 * Parse and add any header link relations to mf data.
196 * @param {Object} microformat
197 * @param {Object} response
199 _mergeLinkHeader(microformat
, response
) {
200 const _scope
= _fileScope('_mergeLinkHeader');
202 // Establish that microformat has expected structure
203 ['rels', 'rel-urls'].forEach((p
) => {
204 if (!(p
in microformat
)) {
205 microformat
[p
] = {}; // eslint-disable-line security/detect-object-injection
208 if (!('items' in microformat
)) {
209 microformat
.items
= [];
212 const linkHeader
= response
.headers
[Enum
.Header
.Link
.toLowerCase()];
216 links
.push(...parseLinkHeader(linkHeader
));
218 this.logger
.error(_scope
, 'failed to parse link header', { error: e
, linkHeader
});
223 // Push header link rels into microformat form.
224 // Inserted at front of lists, as headers take precedence.
225 links
.forEach((link
) => {
226 link
.attributes
.forEach((attr
) => {
227 if (attr
.name
=== 'rel') {
228 if (!(attr
.value
in microformat
.rels
)) {
229 microformat
.rels
[attr
.value
] = [];
231 microformat
.rels
[attr
.value
].unshift(link
.target
);
233 if (!(link
.target
in microformat
['rel-urls'])) {
234 microformat
['rel-urls'][link
.target
] = {
239 microformat
['rel-urls'][link
.target
].rels
.unshift(attr
.value
);
247 * Retrieve and parse microformat data from url.
248 * N.B. this absorbs any errors!
249 * @param {URL} urlObj
252 async
fetchMicroformat(urlObj
) {
253 const _scope
= _fileScope('fetchMicroformat');
254 const logInfoData
= {
256 microformat: undefined,
261 const fetchMicroformatConfig
= Communication
._axiosConfig('GET', urlObj
);
262 response
= await
this.axios(fetchMicroformatConfig
);
264 this.logger
.error(_scope
, 'microformat request failed', { error: e
, ...logInfoData
});
267 logInfoData
.response
= common
.axiosResponseLogData(response
);
269 // Normalize to utf8.
270 let body
= response
.data
;
271 const contentType
= Communication
._parseContentType(response
.headers
[Enum
.Header
.ContentType
.toLowerCase()]);
272 const nonUTF8Charset
= !/utf-*8/i.test(contentType
.params
.charset
) && contentType
.params
.charset
;
273 if (nonUTF8Charset
) {
275 const iconv
= new Iconv(nonUTF8Charset
, 'utf-8//translit//ignore');
276 body
= iconv
.convert(body
).toString('utf8');
278 // istanbul ignore next
279 this.logger
.error(_scope
, 'iconv conversion error', { error: e
, ...logInfoData
});
280 // Try to carry on, maybe the encoding will work anyhow...
284 let microformat
= {};
286 microformat
= mf2(body
, {
287 baseUrl: Communication
._baseUrlString(urlObj
),
290 this.logger
.error(_scope
, 'failed to parse microformat data', { error: e
, ...logInfoData
});
291 // Try to carry on, maybe there are link headers...
294 this._mergeLinkHeader(microformat
, response
);
296 logInfoData
.microformat
= microformat
;
298 this.logger
.debug(_scope
, 'parsed microformat data', logInfoData
);
304 * Retrieve and parse JSON.
305 * N.B. this absorbs any errors!
306 * @param {URL} urlObj
309 async
fetchJSON(urlObj
) {
310 const _scope
= _fileScope('fetchJSON');
311 const logInfoData
= {
317 const fetchJSONConfig
= Communication
._axiosConfig('GET', urlObj
, undefined, undefined, {
318 [Enum
.Header
.Accept
]: [Enum
.ContentType
.ApplicationJson
, Enum
.ContentType
.Any
+ ';q=0.1'].join(', '),
320 response
= await
this.axios(fetchJSONConfig
);
322 this.logger
.error(_scope
, 'json request failed', { error: e
, ...logInfoData
});
325 logInfoData
.response
= common
.axiosResponseLogData(response
);
329 data
= JSON
.parse(response
.data
);
331 this.logger
.error(_scope
, 'json parsing failed', { error: e
, ...logInfoData
});
339 * Validate a url has a specific schema.
340 * @param {URL} urlObj
341 * @param {String[]} validSchemes
343 static _urlValidScheme(urlObj
, validSchemes
= ['http:', 'https:']) {
344 if (!validSchemes
.includes(urlObj
.protocol
)) {
345 throw new ValidationError(`unsupported url scheme '${urlObj.protocol}'`);
351 * Validate a url does not include some components.
352 * @param {URL} urlObj
353 * @param {String[]} disallowed
355 static _urlPartsDisallowed(urlObj
, disallowed
) {
356 disallowed
.forEach((part
) => {
357 if (urlObj
[part
]) { // eslint-disable-line security/detect-object-injection
358 throw new ValidationError(`url cannot contain ${common.properURLComponentName(part)}`);
365 * Validate a url does not have relative path.
366 * @param {String} url
368 static _urlPathNoDots(url
) {
369 if (noDotPathRE
.test(url
)) {
370 throw new ValidationError('relative path segment not valid');
376 * Validate a url does not have a hostname which is an ip address.
377 * N.B. Sets isLoopback on urlObj
378 * @param {URL} urlObj
379 * @param {Boolean} allowLoopback
381 static async
_urlNamedHost(urlObj
, allowLoopback
, resolveHostname
) {
383 if (v6HostRE
.test(urlObj
.hostname
)) {
385 * We do not need to worry about the Address6() failing to parse,
386 * as if it looks like an ipv6 addr but is not valid, the URL()
387 * call would already have failed.
389 address
= new Address6(urlObj
.hostname
.slice(1, urlObj
.hostname
.length
- 1));
390 /* succeeded parsing as ipv6, reject unless loopback */
391 urlObj
.isLoopback
= address
.isLoopback();
394 address
= new Address4(urlObj
.hostname
);
395 /* succeeded parsing as ipv4, reject unless loopback */
396 urlObj
.isLoopback
= address
.isInSubnet(loopback4
);
398 /* did not parse as ip, carry on */
402 if (resolveHostname
&& !urlObj
.isLoopback
) {
404 * Resolve hostname to check for localhost.
405 * This is more complicated due to SSRF mitigation:
406 * If the hostname does not end with a ., we also resolve that,
407 * and complain if the two resolutions do not match, assuming
408 * malicious intent for the server to resolve a local record.
410 const hostnames
= [urlObj
.hostname
];
411 if (!urlObj
.hostname
.endsWith('.')) {
412 hostnames
.push(urlObj
.hostname
+ '.');
414 const settledResolutions
= await Promise
.allSettled(hostnames
.map((hostname
) => dns
.lookupAsync(hostname
, {
418 // If any resolution failed, bail.
419 if (settledResolutions
420 .map((resolution
) => resolution
.status
)
421 .includes('rejected')) {
422 throw new ValidationError('could not resolve hostname');
425 // extract each resolution value, array of {address,family}
426 const resolutions
= settledResolutions
.map((resolution
) => resolution
.value
);
428 // If there were two resolutions, ensure they returned identical results.
429 if (resolutions
.length
> 1) {
430 // create set of addresses for each resolution
431 const addressSets
= resolutions
.map((addrs
) => {
432 return new Set((addrs
|| []).map((a
) => a
.address
));
434 const differences
= common
.setSymmetricDifference(...addressSets
);
435 if (differences
.size
) {
436 throw new ValidationError('inconsistent hostname resolution');
439 const resolvedHost
= resolutions
[0] || [];
441 // Persist the loopback state
442 urlObj
.isLoopback
= resolvedHost
.reduce((acc
, resolved
) => {
444 switch (resolved
.family
) {
446 addr
= new Address4(resolved
.address
);
447 return acc
|| addr
.isInSubnet(loopback4
);
449 addr
= new Address6(resolved
.address
);
450 return acc
|| addr
.isLoopback();
458 && (!urlObj
.isLoopback
|| !allowLoopback
)) {
459 throw new ValidationError('hostname cannot be IP');
465 * Ensure a url meets the requirements to be a profile uri.
466 * @param {String} url
467 * @param {Object} validationOptions
468 * @param {Boolean} validationOptions.allowLoopback
470 async
validateProfile(url
, validationOptions
) {
471 const _scope
= _fileScope('validateProfile');
472 const errorScope
= 'invalid profile url';
474 const options
= Object
.assign({}, {
475 allowLoopback: false,
476 resolveHostname: false,
477 }, validationOptions
);
481 profile
= new URL(url
);
483 this.logger
.debug(_scope
, 'failed to parse url', { error: e
, url
});
484 throw new ValidationError(`${errorScope}: unparsable`);
486 profile
.isLoopback
= false;
489 Communication
._urlValidScheme(profile
);
490 Communication
._urlPartsDisallowed(profile
, ['hash', 'username', 'password', 'port']);
491 Communication
._urlPathNoDots(url
);
492 Communication
._urlNamedHost(profile
, options
.allowLoopback
, options
.resolveHostname
);
494 this.logger
.debug(_scope
, 'profile url not valid', { url
, error: e
});
495 throw new ValidationError(`${errorScope}: ${e.message}`);
503 * Ensure a url meets the requirements to be a client identifier.
504 * Sets 'isLoopback' on returned URL object to true if hostname is or resolves to a loopback ip.
505 * @param {String} url
506 * @param {Object} validationOptions
507 * @param {Boolean} validationOptions.allowLoopback
508 * @param {Boolean} validationOptions.resolveHostname
511 async
validateClientIdentifier(url
, validationOptions
) {
512 const _scope
= _fileScope('validateClientIdentifier');
513 const errorScope
= 'invalid client identifier url';
515 const options
= Object
.assign({}, {
517 resolveHostname: true,
518 }, validationOptions
);
522 clientId
= new URL(url
);
524 this.logger
.debug(_scope
, 'failed to parse url', { error: e
, url
});
525 throw new ValidationError('invalid client identifier url: unparsable');
527 clientId
.isLoopback
= false;
530 Communication
._urlValidScheme(clientId
);
531 Communication
._urlPartsDisallowed(clientId
, ['hash', 'username', 'password']);
532 Communication
._urlPathNoDots(url
);
533 await Communication
._urlNamedHost(clientId
, options
.allowLoopback
, options
.resolveHostname
);
535 this.logger
.debug(_scope
, 'client identifier url not valid', { url
, error: e
});
536 throw new ValidationError(`${errorScope}: ${e.message}`);
544 * @typedef {Object} ClientIdentifierData
545 * @property {Object} rels - keyed by relation to array of uris
546 * @property {HAppData[]} items
549 * Retrieve and parse client identifier endpoint data.
550 * N.B. Assumes urlObj has passed validateClientIdentifier.
551 * @param {URL} urlObj
552 * @returns {ClientIdentifierData|undefined} mf2 data filtered for h-app items, or undefined if url could not be fetched
554 async
fetchClientIdentifier(urlObj
) {
555 const _scope
= _fileScope('fetchClientIdentifier');
557 // Loopback address will eschew client fetch, return empty data.
558 const isLoopbackResult
= {
563 // Set by validation method in case of loopback ip hostname
564 if (urlObj
.isLoopback
) {
565 return isLoopbackResult
;
568 const mfData
= await
this.fetchMicroformat(urlObj
);
573 // Only return h-app items with matching url field.
575 rels: mfData
.rels
|| {},
576 items: (mfData
.items
|| []).filter((item
) => {
577 let urlMatched
= false;
578 const itemType
= item
.type
|| [];
579 if ((itemType
.includes('h-app') || itemType
.includes('h-x-app'))
580 && (item
.properties
&& item
.properties
.url
)) {
581 item
.properties
.url
.forEach((url
) => {
583 const hUrl
= new URL(url
);
584 if (hUrl
.href
=== urlObj
.href
) {
597 * @typedef ProfileData
598 * @property {String} name
599 * @property {String} photo
600 * @property {String} url
601 * @property {String} email
602 * @property {String} authorizationEndpoint - deprecated, backwards compatibility for 20201126 spec
603 * @property {String} tokenEndpoint - deprecated, backwards compatibility for 20201126 spec
604 * @property {String} indieauthMetadata authorization server metadata endpoint
605 * @property {Object} metadata - authorization server metadata for profile
606 * @property {String} metadata.issuer
607 * @property {String} metadata.authorizationEndpoint
608 * @property {String} metadata.tokenEndpoint
609 * @property {String} metadata.introspectionEndpoint
610 * @property {String} metadata.introspectionEndpointAuthMethodsSupported
611 * @property {String} metadata.revocationEndpoint
612 * @property {String} metadata.revocationEndpointAuthMethodsSupported
613 * @property {String} metadata.scopesSupported
614 * @property {String} metadata.responseTypesSupported
615 * @property {String} metadata.grantTypesSupported
616 * @property {String} metadata.serviceDocumentation
617 * @property {String} metadata.codeChallengeMethodsSupported
618 * @property {String} metadata.authorizationResponseIssParameterSupported
619 * @property {String} metadata.userinfoEndpoint
622 * Fetch the relevant microformat data from profile url h-card information,
623 * and authorization server metadata.
624 * @param {URL} urlObj
625 * @returns {ProfileData} mf2 data filtered for select fields from h-card
627 async
fetchProfile(urlObj
) {
628 const _scope
= _fileScope('fetchProfile');
630 const mfData
= await
this.fetchMicroformat(urlObj
);
639 // Locate h-card mf2 items with url field matching profile url,
640 // and populate profile fields with first-encountered card values.
641 if (mfData
&& 'items' in mfData
) {
642 const hCards
= mfData
.items
.filter((item
) =>
643 item
.type
&& item
.type
.includes('h-card') &&
644 item
.properties
&& item
.properties
.url
&& item
.properties
.url
.includes(urlObj
.href
));
645 hCards
.forEach((hCard
) => {
646 Object
.keys(profile
).forEach((key
) => {
647 if (!profile
[key
] && key
in hCard
.properties
) { // eslint-disable-line security/detect-object-injection
648 profile
[key
] = hCard
.properties
[key
][0]; // eslint-disable-line security/detect-object-injection
654 // Populate legacy mf2 fields from relation links.
655 // These will be overwritten if they also exist in server metadata.
657 authorizationEndpoint: 'authorization_endpoint', // backwards compatibility
658 tokenEndpoint: 'token_endpoint', // backwards compatibility
659 }).forEach(([p
, r
]) => {
660 if (mfData
&& r
in mfData
.rels
) {
661 profile
.metadata
[p
] = profile
[p
] = mfData
.rels
[r
][0]; // eslint-disable-line security/detect-object-injection
665 // Set metadata field.
666 if (mfData
&& 'indieauth-metadata' in mfData
.rels
) {
667 profile
.indieauthMetadata
= mfData
.rels
['indieauth-metadata'][0];
670 // Attempt to populate metadata from authorization server.
671 if (profile
.indieauthMetadata
) {
674 mdURL
= new URL(profile
.indieauthMetadata
);
675 } catch (e
) /* istanbul ignore next */ {
676 this.logger
.error(_scope
, 'invalid authorization server metadata url', { profile
});
678 /* istanbul ignore else */
680 const metadataResponse
= await
this.fetchJSON(mdURL
);
681 if (metadataResponse
) {
682 // Map snake_case fields to camelCase.
685 authorizationEndpoint: 'authorization_endpoint',
686 tokenEndpoint: 'token_endpoint',
687 introspectionEndpoint: 'introspection_endpoint',
688 introspectionEndpointAuthMethodsSupported: 'introspection_endpoint_auth_methods_supported',
689 revocationEndpoint: 'revocation_endpoint',
690 revocationEndpointAuthMethodsSupported: 'revocation_endpoint_auth_methods_supported',
691 scopesSupported: 'scopes_supported',
692 responseTypesSupported: 'response_types_supported',
693 grantTypesSupported: 'grant_types_supported',
694 serviceDocumentation: 'service_documentation',
695 codeChallengeMethodsSupported: 'code_challenge_methods_supported',
696 authorizationResponseIssParameterSupported: 'authorization_response_iss_parameter_supported',
697 userinfoEndpoint: 'userinfo_endpoint',
698 }).forEach(([c
, s
]) => {
699 if (s
in metadataResponse
) {
700 profile
.metadata
[c
] = metadataResponse
[s
]; // eslint-disable-line security/detect-object-injection
704 // Populate legacy profile fields.
705 ['authorizationEndpoint', 'tokenEndpoint'].forEach((f
) => {
706 if (f
in profile
.metadata
) {
707 profile
[f
] = profile
.metadata
[f
]; // eslint-disable-line security/detect-object-injection
719 * POST to the auth endpoint, to redeem a code for a profile object.
720 * @param {URL} urlObj
721 * @param {String} code
722 * @param {String} codeVerifier
723 * @param {String} clientId
724 * @param {String} redirectURI
727 async
redeemProfileCode(urlObj
, code
, codeVerifier
, clientId
, redirectURI
) {
728 const _scope
= _fileScope('redeemProfileCode');
730 const data
= new URLSearchParams();
732 'grant_type': 'authorization_code',
734 'client_id': clientId
,
735 'redirect_uri': redirectURI
,
736 'code_verifier': codeVerifier
,
737 }).forEach(([name
, value
]) => data
.set(name
, value
));
739 const postRedeemProfileCodeConfig
= Communication
._axiosConfig('POST', urlObj
, data
.toString(), {}, {
740 [Enum
.Header
.ContentType
]: Enum
.ContentType
.ApplicationForm
,
741 [Enum
.Header
.Accept
]: `${Enum.ContentType.ApplicationJson}, ${Enum.ContentType.Any};q=0.1`,
745 const response
= await
this.axios(postRedeemProfileCodeConfig
);
747 return JSON
.parse(response
.data
);
749 this.logger
.error(_scope
, 'failed to parse json', { error: e
, response
});
753 this.logger
.error(_scope
, 'redeem profile code request failed', { error: e
, url: urlObj
.href
});
760 module
.exports
= Communication
;