add option to persist response body in context for HEAD requests
[squeep-api-dingus] / lib / dingus.js
1 /* eslint-disable security/detect-object-injection */
2 'use strict';
3
4 /**
5 * A very minimal API server framework.
6 * Just a self-contained router and some request glue.
7 */
8
9 require('./patches');
10 const { promises: fsPromises } = require('fs');
11 const path = require('path');
12 const querystring = require('querystring');
13 const common = require('./common');
14 const ContentNegotiation = require('./content-negotiation');
15 const Enum = require('./enum');
16 const { DingusError, ResponseError } = require('./errors');
17 const { extensionToMime } = require('./mime-helper');
18 const Router = require('./router');
19 const Template = require('./template');
20
21 // For logging.
22 const _fileScope = common.fileScope(__filename);
23
24 const defaultOptions = {
25 ignoreTrailingSlash: false,
26 proxyPrefix: '',
27 strictAccept: true,
28 selfBaseUrl: '',
29 staticMetadata: true,
30 staticPath: undefined, // no reasonable default
31 trustProxy: true,
32 querystring,
33 };
34
35 class Dingus {
36 /**
37 * @param {Object} logger object which implements logging methods
38 * @param {Object} options
39 * @param {Boolean} options.ignoreTrailingSlash
40 * @param {string} options.proxyPrefix leading part of url path to strip
41 * @param {Boolean} options.strictAccept whether to error on unsupported Accept type
42 * @param {string} options.selfBaseUrl for constructing links
43 * @param {Boolean} options.staticMetadata serve static headers with static files
44 * @param {Boolean} options.trustProxy trust some header data to be provided by proxy
45 * @param {Object} options.querystring alternate qs parser to use
46 */
47 constructor(logger = common.nullLogger, options = {}) {
48 common.setOptions(this, defaultOptions, options);
49
50 this.router = new Router(options);
51
52 if (!this.proxyPrefix) {
53 this._stripPrefix = (p) => p;
54 }
55
56 this.responseTypes = [
57 Enum.ContentType.TextHTML,
58 Enum.ContentType.TextPlain,
59 Enum.ContentType.ApplicationJson,
60 ];
61
62 this.logger = logger;
63 common.ensureLoggerLevels(this.logger);
64 }
65
66
67 /**
68 * Resolve relative and empty paths in url
69 * @param {string} p path
70 */
71 _normalizePath(p) {
72 const pathNorm = path.normalize(p); // This isn't perfectly correct, but it's easy...
73 return this._stripPrefix(pathNorm);
74 }
75
76
77 /**
78 * Remove a leading portion of url path
79 * N.B. This method gets obliterated if there is no prefix defined at construction
80 * @param {string} p path
81 */
82 _stripPrefix(p) {
83 if (p.startsWith(this.proxyPrefix)) {
84 return p.slice(this.proxyPrefix.length);
85 }
86 return p;
87 }
88
89
90 /**
91 * Returns the path part, and querystring object, from a request url.
92 * @param {string} url
93 */
94 _splitUrl(url) {
95 const [ p, qs ] = common.splitFirst(url, '?');
96 return {
97 pathPart: this._normalizePath(p),
98 queryParams: this.querystring.parse(qs),
99 };
100 }
101
102
103 /**
104 * Insert a new path handler
105 * @param {string} method
106 * @param {string} urlPath
107 * @param {fn} handler
108 */
109 on(method, urlPath, handler, ...handlerArgs) {
110 this.router.on(method, urlPath, handler, handlerArgs);
111 }
112
113
114 /**
115 * Common header tagging for all requests.
116 * Add our own identifier, and persist any external transit identifiers.
117 * @param {http.ClientRequest} req
118 * @param {http.ServerResponse} res
119 * @param {object} ctx
120 */
121 static tagContext(req, res, ctx) {
122 const requestId = common.requestId();
123 ctx.requestId = requestId;
124 res.setHeader(Enum.Header.RequestId, requestId);
125 [Enum.Header.XRequestId, Enum.Header.XCorrelationId].forEach((h) => {
126 const v = req.getHeader(h);
127 if (v) {
128 ctx[h.replace(/-/g, '')] = v;
129 res.setHeader(h, v);
130 }
131 });
132 return requestId;
133 }
134
135
136 /**
137 *
138 * @param {http.ClientRequest} req
139 */
140 _getAddress(req) {
141 // TODO: RFC7239 Forwarded support
142 const address = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedFor)) ||
143 (this.trustProxy && req && req.getHeader(Enum.Header.XRealIP)) ||
144 (req && req.connection && req.connection.remoteAddress) ||
145 '';
146 return address.split(/\s*,\s*/u)[0];
147 }
148
149
150 /**
151 *
152 * @param {http.ClientRequest} req
153 */
154 _getProtocol(req) {
155 // TODO: RFC7239 Forwarded support
156 const protocol = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedProto)) ||
157 ((req && req.connection && req.connection.encrypted) ? 'https' : 'http');
158 return protocol.split(/\s*,\s*/u)[0];
159 }
160
161
162 /**
163 *
164 * @param {http.ClientRequest} req
165 * @param {http.ServerResponse} res
166 * @param {object} ctx
167 */
168 clientAddressContext(req, res, ctx) {
169 ctx.clientAddress = this._getAddress(req);
170 ctx.clientProtocol = this._getProtocol(req);
171 }
172
173
174 /**
175 * Called before every request handler.
176 * @param {http.ClientRequest} req
177 * @param {http.ServerResponse} res
178 * @param {object} ctx
179 */
180 async preHandler(req, res, ctx) {
181 Dingus.tagContext(req, res, ctx);
182 this.clientAddressContext(req, res, ctx);
183 }
184
185
186 /**
187 * Helper for collecting chunks as array of buffers.
188 * @param {Buffer[]} chunks
189 * @param {string|Buffer} chunk
190 * @param {string} encoding
191 */
192 static pushBufChunk(chunks, chunk, encoding = 'utf8') {
193 if (chunk) {
194 if (typeof chunk === 'string') {
195 chunk = Buffer.from(chunk, encoding);
196 }
197 chunks.push(chunk);
198 }
199 }
200
201
202 /**
203 * Sets ctx.responseBody and calls handler upon res.end().
204 * @param {http.ClientRequest} req
205 * @param {http.ServerResponse} res
206 * @param {object} ctx
207 * @param {*} handler fn(req, res, ctx)
208 */
209 static setEndBodyHandler(req, res, ctx, handler) {
210 const origWrite = res.write.bind(res);
211 const origEnd = res.end.bind(res);
212 const chunks = [];
213 res.write = function (chunk, encoding, ...rest) {
214 Dingus.pushBufChunk(chunks, chunk, encoding);
215 return origWrite(chunk, encoding, ...rest);
216 };
217 res.end = function (data, encoding, ...rest) {
218 Dingus.pushBufChunk(chunks, data, encoding);
219 ctx.responseBody = Buffer.concat(chunks);
220 handler(req, res, ctx);
221 return origEnd(data, encoding, ...rest);
222 };
223 }
224
225
226 /**
227 * Intercept writes for head requests, do not send to client,
228 * but send length, and make body available in context.
229 * N.B. If persisted, ctx.responseBody will be a raw buffer, be aware when logging.
230 * @param {http.ClientRequest} req
231 * @param {http.ServerResponse} res
232 * @param {object} ctx
233 * @param {Boolean} persistResponseBody
234 */
235 static setHeadHandler(req, res, ctx, persistResponseBody = false) {
236 if (req.method === 'HEAD') {
237 const origEnd = res.end.bind(res);
238 const chunks = [];
239 res.write = function (chunk, encoding) {
240 Dingus.pushBufChunk(chunks, chunk, encoding);
241 // No call to original res.write.
242 };
243 res.end = function (data, encoding, ...rest) {
244 Dingus.pushBufChunk(chunks, data, encoding);
245 const responseBody = Buffer.concat(chunks);
246 res.setHeader(Enum.Header.ContentLength, Buffer.byteLength(responseBody));
247 if (persistResponseBody) {
248 ctx.responseBody = responseBody;
249 }
250 return origEnd(undefined, encoding, ...rest);
251 };
252 }
253 }
254
255
256 /**
257 * Dispatch the handler for a request
258 * @param {http.ClientRequest} req
259 * @param {http.ServerResponse} res
260 * @param {object} ctx
261 */
262 async dispatch(req, res, ctx = {}) {
263 const _scope = _fileScope('dispatch');
264
265 const { pathPart, queryParams } = this._splitUrl(req.url);
266 ctx.queryParams = queryParams;
267
268 let handler, handlerArgs = [];
269 try {
270 ({ handler, handlerArgs } = this.router.lookup(req.method, pathPart, ctx));
271 } catch (e) {
272 if (e instanceof DingusError) {
273 switch (e.message) {
274 case 'NoPath':
275 handler = this.handlerNotFound.bind(this);
276 break;
277 case 'NoMethod':
278 handler = this.handlerMethodNotAllowed.bind(this);
279 break;
280 default:
281 this.logger.error(_scope, 'unknown dingus error', { error: e });
282 handler = this.handlerInternalServerError.bind(this);
283 }
284 } else if (e instanceof URIError) {
285 handler = this.handlerBadRequest.bind(this);
286 } else {
287 this.logger.error(_scope, 'lookup failure', { error: e });
288 handler = this.handlerInternalServerError.bind(this);
289 }
290 }
291
292 try {
293 await this.preHandler(req, res, ctx);
294 return await handler(req, res, ctx, ...handlerArgs);
295 } catch (e) {
296 ctx.error = e;
297 this.sendErrorResponse(e, req, res, ctx);
298 }
299 }
300
301
302 /**
303 * Return normalized type, without any parameters.
304 * @param {http.ClientRequest} req
305 * @returns {string}
306 */
307 static getRequestContentType(req) {
308 const contentType = req.getHeader(Enum.Header.ContentType);
309 return (contentType || '').split(';')[0].trim().toLowerCase();
310 }
311
312
313 /**
314 * Parse rawBody from ctx as contentType into parsedBody.
315 * @param {string} contentType
316 * @param {object} ctx
317 */
318 parseBody(contentType, ctx) {
319 const _scope = _fileScope('parseBody');
320
321 switch (contentType) {
322 case Enum.ContentType.ApplicationForm:
323 ctx.parsedBody = this.querystring.parse(ctx.rawBody);
324 break;
325
326 case Enum.ContentType.ApplicationJson:
327 try {
328 ctx.parsedBody = JSON.parse(ctx.rawBody);
329 } catch (e) {
330 this.logger.debug(_scope, 'JSON parse failed', { requestId: ctx.requestId, error: e });
331 throw new ResponseError(Enum.ErrorResponse.BadRequest, e.message);
332 }
333 break;
334
335 default:
336 this.logger.debug(_scope, 'unhandled content-type', { requestId: ctx.requestId, contentType });
337 throw new ResponseError(Enum.ErrorResponse.UnsupportedMediaType);
338 }
339 }
340
341
342 /**
343 * Return all body data from a request.
344 * @param {http.ClientRequest} req
345 * @param {Number=} maximumBodySize
346 */
347 async bodyData(req, maximumBodySize) {
348 const _scope = _fileScope('bodyData');
349 return new Promise((resolve, reject) => {
350 const body = [];
351 let length = 0;
352 req.on('data', (chunk) => {
353 body.push(chunk);
354 length += Buffer.byteLength(chunk);
355 if (maximumBodySize && length > maximumBodySize) {
356 this.logger.debug(_scope, 'body data exceeded limit', { length, maximumBodySize });
357 reject(new ResponseError(Enum.ErrorResponse.RequestEntityTooLarge));
358 }
359 });
360 req.on('end', () => resolve(Buffer.concat(body).toString()));
361 req.on('error', (e) => {
362 this.logger.error(_scope, 'failed', { error: e });
363 reject(e);
364 });
365 });
366 }
367
368
369 /**
370 * Read and parse request body data.
371 * @param {http.ClientRequest} req
372 * @param {http.ServerResponse} res
373 * @param {object} ctx
374 */
375 async ingestBody(req, res, ctx) {
376 ctx.rawBody = await this.bodyData(req);
377 const contentType = Dingus.getRequestContentType(req);
378 this.parseBody(contentType, ctx);
379 }
380
381
382 /**
383 * Return the best matching response type.
384 * @param {string[]} responseTypes
385 * @param {http.ClientRequest} req
386 */
387 static getResponseContentType(responseTypes, req) {
388 const acceptHeader = req.getHeader(Enum.Header.Accept);
389 return ContentNegotiation.accept(responseTypes, acceptHeader);
390 }
391
392
393 /**
394 * Returns a list of the most-preferred content encodings for the response.
395 * @param {string[]} responseEncodings
396 * @param {http.ClientRequest} req
397 */
398 static getResponseEncoding(responseEncodings, req) {
399 const acceptEncodingHeader = req.getHeader(Enum.Header.AcceptEncoding);
400 return ContentNegotiation.preferred(responseEncodings, acceptEncodingHeader);
401 }
402
403
404 /**
405 * Set the best content type for the response.
406 * @param {string[]} responseTypes default first
407 * @param {http.ClientRequest} req
408 * @param {http.ServerResponse} res
409 * @param {object} ctx
410 */
411 setResponseType(responseTypes, req, res, ctx) {
412 const _scope = _fileScope('setResponseType');
413 ctx.responseType = Dingus.getResponseContentType(responseTypes, req);
414 if (!ctx.responseType) {
415 if (this.strictAccept) {
416 this.logger.debug(_scope, 'unhandled strict accept', { requestId: req.requestId });
417 throw new ResponseError(Enum.ErrorResponse.NotAcceptable);
418 } else {
419 ctx.responseType = responseTypes[0];
420 }
421 }
422 res.setHeader(Enum.Header.ContentType, ctx.responseType);
423 }
424
425
426 /**
427 * Inserts an encoding
428 * @param {http.ServerResponse} res
429 * @param {string} encoding
430 */
431 static addEncodingHeader(res, encoding) {
432 const existingEncodings = res.getHeader(Enum.Header.ContentEncoding);
433 if (existingEncodings) {
434 encoding = `${encoding}, ${existingEncodings}`;
435 }
436 res.setHeader(Enum.Header.ContentEncoding, encoding);
437 }
438
439
440 /**
441 * Attempt to fetch both data and metadata for a file.
442 * @param {string} filePath
443 */
444 async _readFileInfo(filePath) {
445 const _scope = _fileScope('_readFileInfo');
446 let result;
447 try {
448 // eslint-disable-next-line security/detect-non-literal-fs-filename
449 const stat = fsPromises.stat(filePath);
450 // eslint-disable-next-line security/detect-non-literal-fs-filename
451 const data = fsPromises.readFile(filePath);
452 result = await Promise.all([stat, data]);
453 } catch (e) {
454 if (['ENOENT', 'EACCES', 'EISDIR', 'ENAMETOOLONG', 'EINVAL'].includes(e.code)) {
455 return [null, null];
456 }
457 this.logger.error(_scope, 'fs error', { error: e, filePath });
458 throw e;
459 }
460 return result;
461 }
462
463
464 /**
465 * Potentially add additional headers from static file meta-file.
466 * @param {http.ServerResponse} res
467 * @param {string} directory
468 * @param {string} fileName - already normalized and filtered
469 */
470 async _serveFileMetaHeaders(res, directory, fileName) {
471 const _scope = _fileScope('_serveFileMetaHeaders');
472 this.logger.debug(_scope, 'called', { directory, fileName });
473
474 const metaPrefix = '.';
475 const metaSuffix = '.meta';
476 const metaFileName = `${metaPrefix}${fileName}${metaSuffix}`;
477 const metaFilePath = path.join(directory, metaFileName);
478
479 const [stat, data] = await this._readFileInfo(metaFilePath);
480 if (!stat) {
481 return;
482 }
483
484 const lineBreakRE = /\r\n|\n|\r/;
485 const lines = data.toString().split(lineBreakRE);
486 common.unfoldHeaderLines(lines);
487
488 const headerParseRE = /^(?<name>[^:]+): +(?<value>.*)$/;
489 lines.forEach((line) => {
490 if (line) {
491 const result = headerParseRE.exec(line);
492 const { groups: header } = result;
493 res.setHeader(header.name, header.value);
494 }
495 });
496 }
497
498
499 /**
500 * Serve a file from a directory, with rudimentary cache awareness.
501 * This will also serve pre-encoded variations if available and requested.
502 * @param {http.ClientRequest} req
503 * @param {http.ServerResponse} res
504 * @param {object} ctx
505 * @param {string} directory
506 * @param {string} fileName
507 */
508 async serveFile(req, res, ctx, directory, fileName) {
509 const _scope = _fileScope('serveFile');
510 this.logger.debug(_scope, 'called', { req: common.requestLogData(req), ctx });
511
512 // Require a directory field.
513 if (!directory) {
514 this.logger.debug(_scope, 'rejected unset directory', { fileName });
515 return this.handlerNotFound(req, res, ctx);
516 }
517
518 // Normalize the supplied path, as encoded path-navigation may have been (maliciously) present.
519 fileName = path.normalize(fileName);
520
521 // We will not deal with any subdirs, nor any dot-files.
522 // (Note that we could not deal with subdirs even if we wanted, due to simple router matching scheme.)
523 if (fileName.indexOf(path.sep) >= 0
524 || fileName.charAt(0) === '.') {
525 this.logger.debug(_scope, 'rejected filename', { fileName });
526 return this.handlerNotFound(req, res, ctx);
527 }
528
529 const filePath = path.join(directory, fileName);
530
531 // File must exist, before any alternate static encodings will be considered.
532 let [stat, data] = await this._readFileInfo(filePath);
533 if (!stat) {
534 return this.handlerNotFound(req, res, ctx);
535 }
536
537 // If encodings were requested, check for static versions to serve.
538 // Update stat and data if matching version is found.
539 ctx.availableEncodings = Dingus.getResponseEncoding(Object.values(Enum.EncodingType), req);
540 if (ctx.availableEncodings.length === 0) {
541 // Identity encoding was specifically denied, and nothing else available.
542 this.logger.debug(_scope, 'no suitable encodings', { ctx });
543 return this.handlerMethodNotAllowed(req, res, ctx);
544 }
545 for (const encoding of ctx.availableEncodings) {
546 if (encoding === Enum.EncodingType.Identity) {
547 break;
548 }
549 const suffix = Enum.EncodingTypeSuffix[encoding];
550 if (suffix) {
551 const encodedFilePath = `${filePath}${suffix}`;
552 const [ encodedStat, encodedData ] = await this._readFileInfo(encodedFilePath);
553 if (encodedStat) {
554 ([ stat, data ] = [ encodedStat, encodedData ]);
555 ctx.selectedEncoding = encoding;
556 Dingus.addEncodingHeader(res, encoding);
557 res.setHeader(Enum.Header.Vary, Enum.Header.AcceptEncoding);
558 this.logger.debug(_scope, 'serving encoded version', { ctx, encodedFilePath });
559 }
560 break;
561 }
562 }
563
564 const lastModifiedDate = new Date(stat.mtimeMs);
565 res.setHeader(Enum.Header.LastModified, lastModifiedDate.toGMTString());
566
567 const eTag = common.generateETag(filePath, stat, data);
568 res.setHeader(Enum.Header.ETag, eTag);
569
570 if (common.isClientCached(req, stat.mtimeMs, eTag)) {
571 this.logger.debug(_scope, 'client cached file', { filePath });
572 res.statusCode = 304; // Not Modified
573 res.end();
574 return;
575 }
576
577 // Set the type based on extension of un-encoded filename.
578 const ext = path.extname(filePath).slice(1); // Drop the dot
579 const contentType = extensionToMime(ext);
580 res.setHeader(Enum.Header.ContentType, contentType);
581
582 // We presume static files are relatively cacheable.
583 res.setHeader(Enum.Header.CacheControl, 'public');
584
585 if (this.staticMetadata) {
586 await this._serveFileMetaHeaders(res, directory, fileName);
587 }
588
589 this.logger.debug(_scope, 'serving file', { filePath, contentType });
590 res.end(data);
591 }
592
593
594 /**
595 * Return a content-type appropriate rendering of an errorResponse object.
596 * @param {string} type content-type of response
597 * @param {object} err either an Error object, or an error response
598 * @param {number} err.statusCode
599 * @param {string} err.errorMessage
600 * @param {string|string[]} err.details
601 */
602 // eslint-disable-next-line class-methods-use-this
603 renderError(contentType, err) {
604 switch (contentType) {
605 case Enum.ContentType.ApplicationJson:
606 return JSON.stringify(err);
607
608 case Enum.ContentType.TextHTML:
609 return Template.errorHTML(err);
610
611 case Enum.ContentType.TextPlain:
612 default:
613 return [err.errorMessage, err.details].join('\r\n');
614 }
615 }
616
617
618 /**
619 * Send an error response. Terminal.
620 * Logs any non-error-response errors as such.
621 * @param {object} err either an Error object, or an error response
622 * @param {http.ClientRequest} req
623 * @param {http.ServerResponse} res
624 * @param {object} ctx
625 */
626 sendErrorResponse(err, req, res, ctx) {
627 const _scope = _fileScope('sendErrorResponse');
628 let body;
629
630 // Default to a content type if one is not yet present
631 if (!res.hasHeader(Enum.Header.ContentType)) {
632 res.setHeader(Enum.Header.ContentType, Enum.ContentType.TextPlain);
633 }
634
635 if (err && err.statusCode) {
636 res.statusCode = err.statusCode;
637 body = this.renderError(res.getHeader(Enum.Header.ContentType), err);
638 this.logger.debug(_scope, 'handler error', { err, ...common.handlerLogData(req, res, ctx) });
639 } else {
640 res.statusCode = 500;
641 body = this.renderError(res.getHeader(Enum.Header.ContentType), Enum.ErrorResponse.InternalServerError);
642 this.logger.error(_scope, 'handler exception', { err, ...common.handlerLogData(req, res, ctx) });
643 }
644 res.end(body);
645 }
646
647
648 /**
649 * @param {http.ClientRequest} req
650 * @param {http.ServerResponse} res
651 * @param {object} ctx
652 * @param {String} file - override ctx.params.file
653 */
654 async handlerGetStaticFile(req, res, ctx, file) {
655 Dingus.setHeadHandler(req, res, ctx);
656
657 // Set a default response type to handle any errors; will be re-set to serve actual static content type.
658 this.setResponseType(this.responseTypes, req, res, ctx);
659
660 await this.serveFile(req, res, ctx, this.staticPath, file || ctx.params.file);
661 }
662
663
664 /**
665 * @param {http.ClientRequest} req
666 * @param {http.ServerResponse} res
667 * @param {Object} ctx
668 * @param {String} newPath
669 * @param {Number} statusCode
670 */
671 async handlerRedirect(req, res, ctx, newPath, statusCode = 307) {
672 this.setResponseType(this.responseTypes, req, res, ctx);
673 res.setHeader(Enum.Header.Location, newPath);
674 res.statusCode = statusCode;
675 res.end();
676 }
677
678
679 /**
680 * @param {http.ClientRequest} req
681 * @param {http.ServerResponse} res
682 * @param {object} ctx
683 */
684 async handlerMethodNotAllowed(req, res, ctx) {
685 this.setResponseType(this.responseTypes, req, res, ctx);
686 throw new ResponseError(Enum.ErrorResponse.MethodNotAllowed);
687 }
688
689
690 /**
691 * @param {http.ClientRequest} req
692 * @param {http.ServerResponse} res
693 * @param {object} ctx
694 */
695 async handlerNotFound(req, res, ctx) {
696 this.setResponseType(this.responseTypes, req, res, ctx);
697 throw new ResponseError(Enum.ErrorResponse.NotFound);
698 }
699
700
701 /**
702 * @param {http.ClientRequest} req
703 * @param {http.ServerResponse} res
704 * @param {object} ctx
705 */
706 async handlerBadRequest(req, res, ctx) {
707 this.setResponseType(this.responseTypes, req, res, ctx);
708 throw new ResponseError(Enum.ErrorResponse.BadRequest);
709 }
710
711
712 /**
713 * @param {http.ClientRequest} req
714 * @param {http.ServerResponse} res
715 * @param {object} ctx
716 */
717 async handlerInternalServerError(req, res, ctx) {
718 this.setResponseType(this.responseTypes, req, res, ctx);
719 throw new ResponseError(Enum.ErrorResponse.InternalServerError);
720 }
721
722 }
723
724 module.exports = Dingus;