allow additional arguments to be passed to handler functions
[squeep-api-dingus] / lib / dingus.js
1 /* eslint-disable security/detect-object-injection */
2 'use strict';
3
4 /**
5 * A very minimal API server framework.
6 * Just a self-contained router and some request glue.
7 */
8
9 require('./patches');
10 const { promises: fsPromises } = require('fs');
11 const path = require('path');
12 const querystring = require('querystring');
13 const common = require('./common');
14 const ContentNegotiation = require('./content-negotiation');
15 const Enum = require('./enum');
16 const { DingusError, ResponseError } = require('./errors');
17 const { extensionToMime } = require('./mime-helper');
18 const Router = require('./router');
19 const Template = require('./template');
20
21 // For logging.
22 const _fileScope = common.fileScope(__filename);
23
24 const defaultOptions = {
25 ignoreTrailingSlash: false,
26 proxyPrefix: '',
27 strictAccept: true,
28 selfBaseUrl: '',
29 staticMetadata: true,
30 trustProxy: true,
31 querystring,
32 };
33
34 class Dingus {
35 /**
36 * @param {Object} logger object which implements logging methods
37 * @param {Object} options
38 * @param {Boolean} options.ignoreTrailingSlash
39 * @param {string} options.proxyPrefix leading part of url path to strip
40 * @param {Boolean} options.strictAccept whether to error on unsupported Accept type
41 * @param {string} options.selfBaseUrl for constructing links
42 * @param {Boolean} options.staticMetadata serve static headers with static files
43 * @param {Boolean} options.trustProxy trust some header data to be provided by proxy
44 * @param {Object} options.querystring alternate qs parser to use
45 */
46 constructor(logger = common.nullLogger, options = {}) {
47 common.setOptions(this, defaultOptions, options);
48
49 this.router = new Router(options);
50
51 if (!this.proxyPrefix) {
52 this._stripPrefix = (p) => p;
53 }
54
55 this.responseTypes = [
56 Enum.ContentType.TextHTML,
57 Enum.ContentType.TextPlain,
58 Enum.ContentType.ApplicationJson,
59 ];
60
61 this.logger = logger;
62 common.ensureLoggerLevels(this.logger);
63 }
64
65
66 /**
67 * Resolve relative and empty paths in url
68 * @param {string} p path
69 */
70 _normalizePath(p) {
71 const pathNorm = path.normalize(p); // This isn't perfectly correct, but it's easy...
72 return this._stripPrefix(pathNorm);
73 }
74
75
76 /**
77 * Remove a leading portion of url path
78 * N.B. This method gets obliterated if there is no prefix defined at construction
79 * @param {string} p path
80 */
81 _stripPrefix(p) {
82 if (p.startsWith(this.proxyPrefix)) {
83 return p.slice(this.proxyPrefix.length);
84 }
85 return p;
86 }
87
88
89 /**
90 * Returns the path part, and querystring object, from a request url.
91 * @param {string} url
92 */
93 _splitUrl(url) {
94 const [ p, qs ] = common.splitFirst(url, '?');
95 return {
96 pathPart: this._normalizePath(p),
97 queryParams: this.querystring.parse(qs),
98 };
99 }
100
101
102 /**
103 * Insert a new path handler
104 * @param {string} method
105 * @param {string} urlPath
106 * @param {fn} handler
107 */
108 on(method, urlPath, handler, ...handlerArgs) {
109 this.router.on(method, urlPath, handler, handlerArgs);
110 }
111
112
113 /**
114 * Common header tagging for all requests.
115 * Add our own identifier, and persist any external transit identifiers.
116 * @param {http.ClientRequest} req
117 * @param {http.ServerResponse} res
118 * @param {object} ctx
119 */
120 static tagContext(req, res, ctx) {
121 const requestId = common.requestId();
122 ctx.requestId = requestId;
123 res.setHeader(Enum.Header.RequestId, requestId);
124 [Enum.Header.XRequestId, Enum.Header.XCorrelationId].forEach((h) => {
125 const v = req.getHeader(h);
126 if (v) {
127 ctx[h.replace(/-/g, '')] = v;
128 res.setHeader(h, v);
129 }
130 });
131 return requestId;
132 }
133
134
135 /**
136 *
137 * @param {http.ClientRequest} req
138 */
139 _getAddress(req) {
140 // TODO: RFC7239 Forwarded support
141 const address = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedFor)) ||
142 (this.trustProxy && req && req.getHeader(Enum.Header.XRealIP)) ||
143 (req && req.connection && req.connection.remoteAddress) ||
144 '';
145 return address.split(/\s*,\s*/u)[0];
146 }
147
148
149 /**
150 *
151 * @param {http.ClientRequest} req
152 */
153 _getProtocol(req) {
154 // TODO: RFC7239 Forwarded support
155 const protocol = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedProto)) ||
156 ((req && req.connection && req.connection.encrypted) ? 'https' : 'http');
157 return protocol.split(/\s*,\s*/u)[0];
158 }
159
160
161 /**
162 *
163 * @param {http.ClientRequest} req
164 * @param {http.ServerResponse} res
165 * @param {object} ctx
166 */
167 clientAddressContext(req, res, ctx) {
168 ctx.clientAddress = this._getAddress(req);
169 ctx.clientProtocol = this._getProtocol(req);
170 }
171
172
173 /**
174 * Called before every request handler.
175 * @param {http.ClientRequest} req
176 * @param {http.ServerResponse} res
177 * @param {object} ctx
178 */
179 async preHandler(req, res, ctx) {
180 Dingus.tagContext(req, res, ctx);
181 this.clientAddressContext(req, res, ctx);
182 }
183
184
185 /**
186 * Helper for collecting chunks as array of buffers.
187 * @param {Buffer[]} chunks
188 * @param {string|Buffer} chunk
189 * @param {string} encoding
190 */
191 static pushBufChunk(chunks, chunk, encoding = 'utf8') {
192 if (chunk) {
193 if (typeof chunk === 'string') {
194 chunk = Buffer.from(chunk, encoding);
195 }
196 chunks.push(chunk);
197 }
198 }
199
200
201 /**
202 * Sets ctx.responseBody and calls handler upon res.end().
203 * @param {http.ClientRequest} req
204 * @param {http.ServerResponse} res
205 * @param {object} ctx
206 * @param {*} handler fn(req, res, ctx)
207 */
208 static setEndBodyHandler(req, res, ctx, handler) {
209 const origWrite = res.write.bind(res);
210 const origEnd = res.end.bind(res);
211 const chunks = [];
212 res.write = function (chunk, encoding, ...rest) {
213 Dingus.pushBufChunk(chunks, chunk, encoding);
214 return origWrite(chunk, encoding, ...rest);
215 };
216 res.end = function (data, encoding, ...rest) {
217 Dingus.pushBufChunk(chunks, data, encoding);
218 ctx.responseBody = Buffer.concat(chunks);
219 handler(req, res, ctx);
220 return origEnd(data, encoding, ...rest);
221 };
222 }
223
224
225 /**
226 * Intercept writes for head requests, do not send to client,
227 * but send length, and make body available in context.
228 * @param {http.ClientRequest} req
229 * @param {http.ServerResponse} res
230 * @param {object} ctx
231 */
232 static setHeadHandler(req, res, ctx) {
233 if (req.method === 'HEAD') {
234 const origEnd = res.end.bind(res);
235 const chunks = [];
236 res.write = function (chunk, encoding) {
237 Dingus.pushBufChunk(chunks, chunk, encoding);
238 // No call to original res.write.
239 };
240 res.end = function (data, encoding, ...rest) {
241 Dingus.pushBufChunk(chunks, data, encoding);
242 ctx.responseBody = Buffer.concat(chunks);
243 res.setHeader(Enum.Header.ContentLength, Buffer.byteLength(ctx.responseBody));
244 return origEnd(undefined, encoding, ...rest);
245 };
246 }
247 }
248
249
250 /**
251 * Dispatch the handler for a request
252 * @param {http.ClientRequest} req
253 * @param {http.ServerResponse} res
254 * @param {object} ctx
255 */
256 async dispatch(req, res, ctx = {}) {
257 const _scope = _fileScope('dispatch');
258
259 const { pathPart, queryParams } = this._splitUrl(req.url);
260 ctx.queryParams = queryParams;
261
262 let handler, handlerArgs = [];
263 try {
264 ({ handler, handlerArgs } = this.router.lookup(req.method, pathPart, ctx));
265 } catch (e) {
266 if (e instanceof DingusError) {
267 switch (e.message) {
268 case 'NoPath':
269 handler = this.handlerNotFound.bind(this);
270 break;
271 case 'NoMethod':
272 handler = this.handlerMethodNotAllowed.bind(this);
273 break;
274 default:
275 this.logger.error(_scope, 'unknown dingus error', { error: e });
276 handler = this.handlerInternalServerError.bind(this);
277 }
278 } else if (e instanceof URIError) {
279 handler = this.handlerBadRequest.bind(this);
280 } else {
281 this.logger.error(_scope, 'lookup failure', { error: e });
282 handler = this.handlerInternalServerError.bind(this);
283 }
284 }
285
286 try {
287 await this.preHandler(req, res, ctx);
288 return await handler(req, res, ctx, ...handlerArgs);
289 } catch (e) {
290 ctx.error = e;
291 this.sendErrorResponse(e, req, res, ctx);
292 }
293 }
294
295
296 /**
297 * Return normalized type, without any parameters.
298 * @param {http.ClientRequest} req
299 * @returns {string}
300 */
301 static getRequestContentType(req) {
302 const contentType = req.getHeader(Enum.Header.ContentType);
303 return (contentType || '').split(';')[0].trim().toLowerCase();
304 }
305
306
307 /**
308 * Parse rawBody from ctx as contentType into parsedBody.
309 * @param {string} contentType
310 * @param {object} ctx
311 */
312 parseBody(contentType, ctx) {
313 const _scope = _fileScope('parseBody');
314
315 switch (contentType) {
316 case Enum.ContentType.ApplicationForm:
317 ctx.parsedBody = this.querystring.parse(ctx.rawBody);
318 break;
319
320 case Enum.ContentType.ApplicationJson:
321 try {
322 ctx.parsedBody = JSON.parse(ctx.rawBody);
323 } catch (e) {
324 this.logger.debug(_scope, 'JSON parse failed', { requestId: ctx.requestId, error: e });
325 throw new ResponseError(Enum.ErrorResponse.BadRequest, e.message);
326 }
327 break;
328
329 default:
330 this.logger.debug(_scope, 'unhandled content-type', { requestId: ctx.requestId, contentType });
331 throw new ResponseError(Enum.ErrorResponse.UnsupportedMediaType);
332 }
333 }
334
335
336 /**
337 * Return all body data from a request.
338 * @param {http.ClientRequest} req
339 */
340 async bodyData(req) {
341 const _scope = _fileScope('bodyData');
342 return new Promise((resolve, reject) => {
343 const body = [];
344 req.on('data', (chunk) => body.push(chunk));
345 req.on('end', () => resolve(Buffer.concat(body).toString()));
346 req.on('error', (e) => {
347 this.logger.error(_scope, 'failed', { error: e });
348 reject(e);
349 });
350 });
351 }
352
353
354 /**
355 * Read and parse request body data.
356 * @param {http.ClientRequest} req
357 * @param {http.ServerResponse} res
358 * @param {object} ctx
359 */
360 async ingestBody(req, res, ctx) {
361 ctx.rawBody = await this.bodyData(req);
362 const contentType = Dingus.getRequestContentType(req);
363 this.parseBody(contentType, ctx);
364 }
365
366
367 /**
368 * Return the best matching response type.
369 * @param {string[]} responseTypes
370 * @param {http.ClientRequest} req
371 */
372 static getResponseContentType(responseTypes, req) {
373 const acceptHeader = req.getHeader(Enum.Header.Accept);
374 return ContentNegotiation.accept(responseTypes, acceptHeader);
375 }
376
377
378 /**
379 * Returns a list of the most-preferred content encodings for the response.
380 * @param {string[]} responseEncodings
381 * @param {http.ClientRequest} req
382 */
383 static getResponseEncoding(responseEncodings, req) {
384 const acceptEncodingHeader = req.getHeader(Enum.Header.AcceptEncoding);
385 return ContentNegotiation.preferred(responseEncodings, acceptEncodingHeader);
386 }
387
388
389 /**
390 * Set the best content type for the response.
391 * @param {string[]} responseTypes default first
392 * @param {http.ClientRequest} req
393 * @param {http.ServerResponse} res
394 * @param {object} ctx
395 */
396 setResponseType(responseTypes, req, res, ctx) {
397 const _scope = _fileScope('setResponseType');
398 ctx.responseType = Dingus.getResponseContentType(responseTypes, req);
399 if (!ctx.responseType) {
400 if (this.strictAccept) {
401 this.logger.debug(_scope, 'unhandled strict accept', { requestId: req.requestId });
402 throw new ResponseError(Enum.ErrorResponse.NotAcceptable);
403 } else {
404 ctx.responseType = responseTypes[0];
405 }
406 }
407 res.setHeader(Enum.Header.ContentType, ctx.responseType);
408 }
409
410
411 /**
412 * Inserts an encoding
413 * @param {http.ServerResponse} res
414 * @param {string} encoding
415 */
416 static addEncodingHeader(res, encoding) {
417 const existingEncodings = res.getHeader(Enum.Header.ContentEncoding);
418 if (existingEncodings) {
419 encoding = `${encoding}, ${existingEncodings}`;
420 }
421 res.setHeader(Enum.Header.ContentEncoding, encoding);
422 }
423
424
425 /**
426 * Attempt to fetch both data and metadata for a file.
427 * @param {string} filePath
428 */
429 async _readFileInfo(filePath) {
430 const _scope = _fileScope('_readFileInfo');
431 let result;
432 try {
433 // eslint-disable-next-line security/detect-non-literal-fs-filename
434 const stat = fsPromises.stat(filePath);
435 // eslint-disable-next-line security/detect-non-literal-fs-filename
436 const data = fsPromises.readFile(filePath);
437 result = await Promise.all([stat, data]);
438 } catch (e) {
439 if (['ENOENT', 'EACCES', 'EISDIR', 'ENAMETOOLONG', 'EINVAL'].includes(e.code)) {
440 return [null, null];
441 }
442 this.logger.error(_scope, 'fs error', { error: e, filePath });
443 throw e;
444 }
445 return result;
446 }
447
448
449 /**
450 * Potentially add additional headers from static file meta-file.
451 * @param {http.ServerResponse} res
452 * @param {string} directory
453 * @param {string} fileName - already normalized and filtered
454 */
455 async _serveFileMetaHeaders(res, directory, fileName) {
456 const _scope = _fileScope('_serveFileMetaHeaders');
457 this.logger.debug(_scope, 'called', { directory, fileName });
458
459 const metaPrefix = '.';
460 const metaSuffix = '.meta';
461 const metaFileName = `${metaPrefix}${fileName}${metaSuffix}`;
462 const metaFilePath = path.join(directory, metaFileName);
463
464 const [stat, data] = await this._readFileInfo(metaFilePath);
465 if (!stat) {
466 return;
467 }
468
469 const lineBreakRE = /\r\n|\n|\r/;
470 const lines = data.toString().split(lineBreakRE);
471 common.unfoldHeaderLines(lines);
472
473 const headerParseRE = /^(?<name>[^:]+): +(?<value>.*)$/;
474 lines.forEach((line) => {
475 if (line) {
476 const result = headerParseRE.exec(line);
477 const { groups: header } = result;
478 res.setHeader(header.name, header.value);
479 }
480 });
481 }
482
483
484 /**
485 * Serve a file from a directory, with rudimentary cache awareness.
486 * This will also serve pre-encoded variations if available and requested.
487 * @param {http.ClientRequest} req
488 * @param {http.ServerResponse} res
489 * @param {object} ctx
490 * @param {string} directory
491 * @param {string} fileName
492 */
493 async serveFile(req, res, ctx, directory, fileName) {
494 const _scope = _fileScope('serveFile');
495 this.logger.debug(_scope, 'called', { req: common.requestLogData(req), ctx });
496
497 // Normalize the supplied path, as encoded path-navigation may have been (maliciously) present.
498 fileName = path.normalize(fileName);
499
500 // We will not deal with any subdirs, nor any dot-files.
501 // (Note that we could not deal with subdirs even if we wanted, due to simple router matching scheme.)
502 if (fileName.indexOf(path.sep) >= 0
503 || fileName.charAt(0) === '.') {
504 this.logger.debug(_scope, 'rejected filename', { fileName });
505 return this.handlerNotFound(req, res, ctx);
506 }
507
508 const filePath = path.join(directory, fileName);
509
510 // File must exist, before any alternate static encodings will be considered.
511 let [stat, data] = await this._readFileInfo(filePath);
512 if (!stat) {
513 return this.handlerNotFound(req, res, ctx);
514 }
515
516 // If encodings were requested, check for static versions to serve.
517 // Update stat and data if matching version is found.
518 ctx.availableEncodings = Dingus.getResponseEncoding(Object.values(Enum.EncodingType), req);
519 if (ctx.availableEncodings.length === 0) {
520 // Identity encoding was specifically denied, and nothing else available.
521 this.logger.debug(_scope, 'no suitable encodings', { ctx });
522 return this.handlerMethodNotAllowed(req, res, ctx);
523 }
524 for (const encoding of ctx.availableEncodings) {
525 if (encoding === Enum.EncodingType.Identity) {
526 break;
527 }
528 const suffix = Enum.EncodingTypeSuffix[encoding];
529 if (suffix) {
530 const encodedFilePath = `${filePath}${suffix}`;
531 const [ encodedStat, encodedData ] = await this._readFileInfo(encodedFilePath);
532 if (encodedStat) {
533 ([ stat, data ] = [ encodedStat, encodedData ]);
534 ctx.selectedEncoding = encoding;
535 Dingus.addEncodingHeader(res, encoding);
536 res.setHeader(Enum.Header.Vary, Enum.Header.AcceptEncoding);
537 this.logger.debug(_scope, 'serving encoded version', { ctx, encodedFilePath });
538 }
539 break;
540 }
541 }
542
543 const lastModifiedDate = new Date(stat.mtimeMs);
544 res.setHeader(Enum.Header.LastModified, lastModifiedDate.toGMTString());
545
546 const eTag = common.generateETag(filePath, stat, data);
547 res.setHeader(Enum.Header.ETag, eTag);
548
549 if (common.isClientCached(req, stat.mtimeMs, eTag)) {
550 this.logger.debug(_scope, 'client cached file', { filePath });
551 res.statusCode = 304; // Not Modified
552 res.end();
553 return;
554 }
555
556 // Set the type based on extension of un-encoded filename.
557 const ext = path.extname(filePath).slice(1); // Drop the dot
558 const contentType = extensionToMime(ext);
559 res.setHeader(Enum.Header.ContentType, contentType);
560
561 // We presume static files are relatively cacheable.
562 res.setHeader(Enum.Header.CacheControl, 'public');
563
564 if (this.staticMetadata) {
565 await this._serveFileMetaHeaders(res, directory, fileName);
566 }
567
568 this.logger.debug(_scope, 'serving file', { filePath, contentType });
569 res.end(data);
570 }
571
572
573 /**
574 * Return a content-type appropriate rendering of an errorResponse object.
575 * @param {string} type content-type of response
576 * @param {object} err either an Error object, or an error response
577 * @param {number} err.statusCode
578 * @param {string} err.errorMessage
579 * @param {string|string[]} err.details
580 */
581 // eslint-disable-next-line class-methods-use-this
582 renderError(contentType, err) {
583 switch (contentType) {
584 case Enum.ContentType.ApplicationJson:
585 return JSON.stringify(err);
586
587 case Enum.ContentType.TextHTML:
588 return Template.errorHTML(err);
589
590 case Enum.ContentType.TextPlain:
591 default:
592 return [err.errorMessage, err.details].join('\r\n');
593 }
594 }
595
596
597 /**
598 * Send an error response. Terminal.
599 * Logs any non-error-response errors as such.
600 * @param {object} err either an Error object, or an error response
601 * @param {http.ClientRequest} req
602 * @param {http.ServerResponse} res
603 * @param {object} ctx
604 */
605 sendErrorResponse(err, req, res, ctx) {
606 const _scope = _fileScope('sendErrorResponse');
607 let body;
608
609 // Default to a content type if one is not yet present
610 if (!res.hasHeader(Enum.Header.ContentType)) {
611 res.setHeader(Enum.Header.ContentType, Enum.ContentType.TextPlain);
612 }
613
614 if (err && err.statusCode) {
615 res.statusCode = err.statusCode;
616 body = this.renderError(res.getHeader(Enum.Header.ContentType), err);
617 this.logger.debug(_scope, 'handler error', { err, ...common.handlerLogData(req, res, ctx) });
618 } else {
619 res.statusCode = 500;
620 body = this.renderError(res.getHeader(Enum.Header.ContentType), Enum.ErrorResponse.InternalServerError);
621 this.logger.error(_scope, 'handler exception', { err, ...common.handlerLogData(req, res, ctx) });
622 }
623 res.end(body);
624 }
625
626
627 /**
628 * @param {http.ClientRequest} req
629 * @param {http.ServerResponse} res
630 * @param {object} ctx
631 * @param {String} file - override ctx.params.file
632 */
633 async handlerGetStaticFile(req, res, ctx, file) {
634 Dingus.setHeadHandler(req, res, ctx);
635
636 // Set a default response type to handle any errors; will be re-set to serve actual static content type.
637 this.setResponseType(this.responseTypes, req, res, ctx);
638
639 await this.serveFile(req, res, ctx, this.staticPath, file || ctx.params.file);
640 }
641
642
643 /**
644 * @param {http.ClientRequest} req
645 * @param {http.ServerResponse} res
646 * @param {Object} ctx
647 * @param {String} newPath
648 * @param {Number} statusCode
649 */
650 async handlerRedirect(req, res, ctx, newPath, statusCode = 307) {
651 this.setResponseType(this.responseTypes, req, res, ctx);
652 res.setHeader(Enum.Header.Location, newPath);
653 res.statusCode = statusCode;
654 res.end();
655 }
656
657
658 /**
659 * @param {http.ClientRequest} req
660 * @param {http.ServerResponse} res
661 * @param {object} ctx
662 */
663 async handlerMethodNotAllowed(req, res, ctx) {
664 this.setResponseType(this.responseTypes, req, res, ctx);
665 throw new ResponseError(Enum.ErrorResponse.MethodNotAllowed);
666 }
667
668
669 /**
670 * @param {http.ClientRequest} req
671 * @param {http.ServerResponse} res
672 * @param {object} ctx
673 */
674 async handlerNotFound(req, res, ctx) {
675 this.setResponseType(this.responseTypes, req, res, ctx);
676 throw new ResponseError(Enum.ErrorResponse.NotFound);
677 }
678
679
680 /**
681 * @param {http.ClientRequest} req
682 * @param {http.ServerResponse} res
683 * @param {object} ctx
684 */
685 async handlerBadRequest(req, res, ctx) {
686 this.setResponseType(this.responseTypes, req, res, ctx);
687 throw new ResponseError(Enum.ErrorResponse.BadRequest);
688 }
689
690
691 /**
692 * @param {http.ClientRequest} req
693 * @param {http.ServerResponse} res
694 * @param {object} ctx
695 */
696 async handlerInternalServerError(req, res, ctx) {
697 this.setResponseType(this.responseTypes, req, res, ctx);
698 throw new ResponseError(Enum.ErrorResponse.InternalServerError);
699 }
700
701 }
702
703 module.exports = Dingus;