support maximum request body size
[squeep-api-dingus] / lib / dingus.js
1 /* eslint-disable security/detect-object-injection */
2 'use strict';
3
4 /**
5 * A very minimal API server framework.
6 * Just a self-contained router and some request glue.
7 */
8
9 require('./patches');
10 const { promises: fsPromises } = require('fs');
11 const path = require('path');
12 const querystring = require('querystring');
13 const common = require('./common');
14 const ContentNegotiation = require('./content-negotiation');
15 const Enum = require('./enum');
16 const { DingusError, ResponseError } = require('./errors');
17 const { extensionToMime } = require('./mime-helper');
18 const Router = require('./router');
19 const Template = require('./template');
20
21 // For logging.
22 const _fileScope = common.fileScope(__filename);
23
24 const defaultOptions = {
25 ignoreTrailingSlash: false,
26 proxyPrefix: '',
27 strictAccept: true,
28 selfBaseUrl: '',
29 staticMetadata: true,
30 staticPath: undefined, // no reasonable default
31 trustProxy: true,
32 querystring,
33 };
34
35 class Dingus {
36 /**
37 * @param {Object} logger object which implements logging methods
38 * @param {Object} options
39 * @param {Boolean} options.ignoreTrailingSlash
40 * @param {string} options.proxyPrefix leading part of url path to strip
41 * @param {Boolean} options.strictAccept whether to error on unsupported Accept type
42 * @param {string} options.selfBaseUrl for constructing links
43 * @param {Boolean} options.staticMetadata serve static headers with static files
44 * @param {Boolean} options.trustProxy trust some header data to be provided by proxy
45 * @param {Object} options.querystring alternate qs parser to use
46 */
47 constructor(logger = common.nullLogger, options = {}) {
48 common.setOptions(this, defaultOptions, options);
49
50 this.router = new Router(options);
51
52 if (!this.proxyPrefix) {
53 this._stripPrefix = (p) => p;
54 }
55
56 this.responseTypes = [
57 Enum.ContentType.TextHTML,
58 Enum.ContentType.TextPlain,
59 Enum.ContentType.ApplicationJson,
60 ];
61
62 this.logger = logger;
63 common.ensureLoggerLevels(this.logger);
64 }
65
66
67 /**
68 * Resolve relative and empty paths in url
69 * @param {string} p path
70 */
71 _normalizePath(p) {
72 const pathNorm = path.normalize(p); // This isn't perfectly correct, but it's easy...
73 return this._stripPrefix(pathNorm);
74 }
75
76
77 /**
78 * Remove a leading portion of url path
79 * N.B. This method gets obliterated if there is no prefix defined at construction
80 * @param {string} p path
81 */
82 _stripPrefix(p) {
83 if (p.startsWith(this.proxyPrefix)) {
84 return p.slice(this.proxyPrefix.length);
85 }
86 return p;
87 }
88
89
90 /**
91 * Returns the path part, and querystring object, from a request url.
92 * @param {string} url
93 */
94 _splitUrl(url) {
95 const [ p, qs ] = common.splitFirst(url, '?');
96 return {
97 pathPart: this._normalizePath(p),
98 queryParams: this.querystring.parse(qs),
99 };
100 }
101
102
103 /**
104 * Insert a new path handler
105 * @param {string} method
106 * @param {string} urlPath
107 * @param {fn} handler
108 */
109 on(method, urlPath, handler, ...handlerArgs) {
110 this.router.on(method, urlPath, handler, handlerArgs);
111 }
112
113
114 /**
115 * Common header tagging for all requests.
116 * Add our own identifier, and persist any external transit identifiers.
117 * @param {http.ClientRequest} req
118 * @param {http.ServerResponse} res
119 * @param {object} ctx
120 */
121 static tagContext(req, res, ctx) {
122 const requestId = common.requestId();
123 ctx.requestId = requestId;
124 res.setHeader(Enum.Header.RequestId, requestId);
125 [Enum.Header.XRequestId, Enum.Header.XCorrelationId].forEach((h) => {
126 const v = req.getHeader(h);
127 if (v) {
128 ctx[h.replace(/-/g, '')] = v;
129 res.setHeader(h, v);
130 }
131 });
132 return requestId;
133 }
134
135
136 /**
137 *
138 * @param {http.ClientRequest} req
139 */
140 _getAddress(req) {
141 // TODO: RFC7239 Forwarded support
142 const address = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedFor)) ||
143 (this.trustProxy && req && req.getHeader(Enum.Header.XRealIP)) ||
144 (req && req.connection && req.connection.remoteAddress) ||
145 '';
146 return address.split(/\s*,\s*/u)[0];
147 }
148
149
150 /**
151 *
152 * @param {http.ClientRequest} req
153 */
154 _getProtocol(req) {
155 // TODO: RFC7239 Forwarded support
156 const protocol = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedProto)) ||
157 ((req && req.connection && req.connection.encrypted) ? 'https' : 'http');
158 return protocol.split(/\s*,\s*/u)[0];
159 }
160
161
162 /**
163 *
164 * @param {http.ClientRequest} req
165 * @param {http.ServerResponse} res
166 * @param {object} ctx
167 */
168 clientAddressContext(req, res, ctx) {
169 ctx.clientAddress = this._getAddress(req);
170 ctx.clientProtocol = this._getProtocol(req);
171 }
172
173
174 /**
175 * Called before every request handler.
176 * @param {http.ClientRequest} req
177 * @param {http.ServerResponse} res
178 * @param {object} ctx
179 */
180 async preHandler(req, res, ctx) {
181 Dingus.tagContext(req, res, ctx);
182 this.clientAddressContext(req, res, ctx);
183 }
184
185
186 /**
187 * Helper for collecting chunks as array of buffers.
188 * @param {Buffer[]} chunks
189 * @param {string|Buffer} chunk
190 * @param {string} encoding
191 */
192 static pushBufChunk(chunks, chunk, encoding = 'utf8') {
193 if (chunk) {
194 if (typeof chunk === 'string') {
195 chunk = Buffer.from(chunk, encoding);
196 }
197 chunks.push(chunk);
198 }
199 }
200
201
202 /**
203 * Sets ctx.responseBody and calls handler upon res.end().
204 * @param {http.ClientRequest} req
205 * @param {http.ServerResponse} res
206 * @param {object} ctx
207 * @param {*} handler fn(req, res, ctx)
208 */
209 static setEndBodyHandler(req, res, ctx, handler) {
210 const origWrite = res.write.bind(res);
211 const origEnd = res.end.bind(res);
212 const chunks = [];
213 res.write = function (chunk, encoding, ...rest) {
214 Dingus.pushBufChunk(chunks, chunk, encoding);
215 return origWrite(chunk, encoding, ...rest);
216 };
217 res.end = function (data, encoding, ...rest) {
218 Dingus.pushBufChunk(chunks, data, encoding);
219 ctx.responseBody = Buffer.concat(chunks);
220 handler(req, res, ctx);
221 return origEnd(data, encoding, ...rest);
222 };
223 }
224
225
226 /**
227 * Intercept writes for head requests, do not send to client,
228 * but send length, and make body available in context.
229 * @param {http.ClientRequest} req
230 * @param {http.ServerResponse} res
231 * @param {object} ctx
232 */
233 static setHeadHandler(req, res, ctx) {
234 if (req.method === 'HEAD') {
235 const origEnd = res.end.bind(res);
236 const chunks = [];
237 res.write = function (chunk, encoding) {
238 Dingus.pushBufChunk(chunks, chunk, encoding);
239 // No call to original res.write.
240 };
241 res.end = function (data, encoding, ...rest) {
242 Dingus.pushBufChunk(chunks, data, encoding);
243 ctx.responseBody = Buffer.concat(chunks);
244 res.setHeader(Enum.Header.ContentLength, Buffer.byteLength(ctx.responseBody));
245 return origEnd(undefined, encoding, ...rest);
246 };
247 }
248 }
249
250
251 /**
252 * Dispatch the handler for a request
253 * @param {http.ClientRequest} req
254 * @param {http.ServerResponse} res
255 * @param {object} ctx
256 */
257 async dispatch(req, res, ctx = {}) {
258 const _scope = _fileScope('dispatch');
259
260 const { pathPart, queryParams } = this._splitUrl(req.url);
261 ctx.queryParams = queryParams;
262
263 let handler, handlerArgs = [];
264 try {
265 ({ handler, handlerArgs } = this.router.lookup(req.method, pathPart, ctx));
266 } catch (e) {
267 if (e instanceof DingusError) {
268 switch (e.message) {
269 case 'NoPath':
270 handler = this.handlerNotFound.bind(this);
271 break;
272 case 'NoMethod':
273 handler = this.handlerMethodNotAllowed.bind(this);
274 break;
275 default:
276 this.logger.error(_scope, 'unknown dingus error', { error: e });
277 handler = this.handlerInternalServerError.bind(this);
278 }
279 } else if (e instanceof URIError) {
280 handler = this.handlerBadRequest.bind(this);
281 } else {
282 this.logger.error(_scope, 'lookup failure', { error: e });
283 handler = this.handlerInternalServerError.bind(this);
284 }
285 }
286
287 try {
288 await this.preHandler(req, res, ctx);
289 return await handler(req, res, ctx, ...handlerArgs);
290 } catch (e) {
291 ctx.error = e;
292 this.sendErrorResponse(e, req, res, ctx);
293 }
294 }
295
296
297 /**
298 * Return normalized type, without any parameters.
299 * @param {http.ClientRequest} req
300 * @returns {string}
301 */
302 static getRequestContentType(req) {
303 const contentType = req.getHeader(Enum.Header.ContentType);
304 return (contentType || '').split(';')[0].trim().toLowerCase();
305 }
306
307
308 /**
309 * Parse rawBody from ctx as contentType into parsedBody.
310 * @param {string} contentType
311 * @param {object} ctx
312 */
313 parseBody(contentType, ctx) {
314 const _scope = _fileScope('parseBody');
315
316 switch (contentType) {
317 case Enum.ContentType.ApplicationForm:
318 ctx.parsedBody = this.querystring.parse(ctx.rawBody);
319 break;
320
321 case Enum.ContentType.ApplicationJson:
322 try {
323 ctx.parsedBody = JSON.parse(ctx.rawBody);
324 } catch (e) {
325 this.logger.debug(_scope, 'JSON parse failed', { requestId: ctx.requestId, error: e });
326 throw new ResponseError(Enum.ErrorResponse.BadRequest, e.message);
327 }
328 break;
329
330 default:
331 this.logger.debug(_scope, 'unhandled content-type', { requestId: ctx.requestId, contentType });
332 throw new ResponseError(Enum.ErrorResponse.UnsupportedMediaType);
333 }
334 }
335
336
337 /**
338 * Return all body data from a request.
339 * @param {http.ClientRequest} req
340 * @param {Number=} maximumBodySize
341 */
342 async bodyData(req, maximumBodySize) {
343 const _scope = _fileScope('bodyData');
344 return new Promise((resolve, reject) => {
345 const body = [];
346 let length = 0;
347 req.on('data', (chunk) => {
348 body.push(chunk);
349 length += Buffer.byteLength(chunk);
350 if (maximumBodySize && length > maximumBodySize) {
351 this.logger.debug(_scope, 'body data exceeded limit', { length, maximumBodySize });
352 reject(new ResponseError(Enum.ErrorResponse.RequestEntityTooLarge));
353 }
354 });
355 req.on('end', () => resolve(Buffer.concat(body).toString()));
356 req.on('error', (e) => {
357 this.logger.error(_scope, 'failed', { error: e });
358 reject(e);
359 });
360 });
361 }
362
363
364 /**
365 * Read and parse request body data.
366 * @param {http.ClientRequest} req
367 * @param {http.ServerResponse} res
368 * @param {object} ctx
369 */
370 async ingestBody(req, res, ctx) {
371 ctx.rawBody = await this.bodyData(req);
372 const contentType = Dingus.getRequestContentType(req);
373 this.parseBody(contentType, ctx);
374 }
375
376
377 /**
378 * Return the best matching response type.
379 * @param {string[]} responseTypes
380 * @param {http.ClientRequest} req
381 */
382 static getResponseContentType(responseTypes, req) {
383 const acceptHeader = req.getHeader(Enum.Header.Accept);
384 return ContentNegotiation.accept(responseTypes, acceptHeader);
385 }
386
387
388 /**
389 * Returns a list of the most-preferred content encodings for the response.
390 * @param {string[]} responseEncodings
391 * @param {http.ClientRequest} req
392 */
393 static getResponseEncoding(responseEncodings, req) {
394 const acceptEncodingHeader = req.getHeader(Enum.Header.AcceptEncoding);
395 return ContentNegotiation.preferred(responseEncodings, acceptEncodingHeader);
396 }
397
398
399 /**
400 * Set the best content type for the response.
401 * @param {string[]} responseTypes default first
402 * @param {http.ClientRequest} req
403 * @param {http.ServerResponse} res
404 * @param {object} ctx
405 */
406 setResponseType(responseTypes, req, res, ctx) {
407 const _scope = _fileScope('setResponseType');
408 ctx.responseType = Dingus.getResponseContentType(responseTypes, req);
409 if (!ctx.responseType) {
410 if (this.strictAccept) {
411 this.logger.debug(_scope, 'unhandled strict accept', { requestId: req.requestId });
412 throw new ResponseError(Enum.ErrorResponse.NotAcceptable);
413 } else {
414 ctx.responseType = responseTypes[0];
415 }
416 }
417 res.setHeader(Enum.Header.ContentType, ctx.responseType);
418 }
419
420
421 /**
422 * Inserts an encoding
423 * @param {http.ServerResponse} res
424 * @param {string} encoding
425 */
426 static addEncodingHeader(res, encoding) {
427 const existingEncodings = res.getHeader(Enum.Header.ContentEncoding);
428 if (existingEncodings) {
429 encoding = `${encoding}, ${existingEncodings}`;
430 }
431 res.setHeader(Enum.Header.ContentEncoding, encoding);
432 }
433
434
435 /**
436 * Attempt to fetch both data and metadata for a file.
437 * @param {string} filePath
438 */
439 async _readFileInfo(filePath) {
440 const _scope = _fileScope('_readFileInfo');
441 let result;
442 try {
443 // eslint-disable-next-line security/detect-non-literal-fs-filename
444 const stat = fsPromises.stat(filePath);
445 // eslint-disable-next-line security/detect-non-literal-fs-filename
446 const data = fsPromises.readFile(filePath);
447 result = await Promise.all([stat, data]);
448 } catch (e) {
449 if (['ENOENT', 'EACCES', 'EISDIR', 'ENAMETOOLONG', 'EINVAL'].includes(e.code)) {
450 return [null, null];
451 }
452 this.logger.error(_scope, 'fs error', { error: e, filePath });
453 throw e;
454 }
455 return result;
456 }
457
458
459 /**
460 * Potentially add additional headers from static file meta-file.
461 * @param {http.ServerResponse} res
462 * @param {string} directory
463 * @param {string} fileName - already normalized and filtered
464 */
465 async _serveFileMetaHeaders(res, directory, fileName) {
466 const _scope = _fileScope('_serveFileMetaHeaders');
467 this.logger.debug(_scope, 'called', { directory, fileName });
468
469 const metaPrefix = '.';
470 const metaSuffix = '.meta';
471 const metaFileName = `${metaPrefix}${fileName}${metaSuffix}`;
472 const metaFilePath = path.join(directory, metaFileName);
473
474 const [stat, data] = await this._readFileInfo(metaFilePath);
475 if (!stat) {
476 return;
477 }
478
479 const lineBreakRE = /\r\n|\n|\r/;
480 const lines = data.toString().split(lineBreakRE);
481 common.unfoldHeaderLines(lines);
482
483 const headerParseRE = /^(?<name>[^:]+): +(?<value>.*)$/;
484 lines.forEach((line) => {
485 if (line) {
486 const result = headerParseRE.exec(line);
487 const { groups: header } = result;
488 res.setHeader(header.name, header.value);
489 }
490 });
491 }
492
493
494 /**
495 * Serve a file from a directory, with rudimentary cache awareness.
496 * This will also serve pre-encoded variations if available and requested.
497 * @param {http.ClientRequest} req
498 * @param {http.ServerResponse} res
499 * @param {object} ctx
500 * @param {string} directory
501 * @param {string} fileName
502 */
503 async serveFile(req, res, ctx, directory, fileName) {
504 const _scope = _fileScope('serveFile');
505 this.logger.debug(_scope, 'called', { req: common.requestLogData(req), ctx });
506
507 // Require a directory field.
508 if (!directory) {
509 this.logger.debug(_scope, 'rejected unset directory', { fileName });
510 return this.handlerNotFound(req, res, ctx);
511 }
512
513 // Normalize the supplied path, as encoded path-navigation may have been (maliciously) present.
514 fileName = path.normalize(fileName);
515
516 // We will not deal with any subdirs, nor any dot-files.
517 // (Note that we could not deal with subdirs even if we wanted, due to simple router matching scheme.)
518 if (fileName.indexOf(path.sep) >= 0
519 || fileName.charAt(0) === '.') {
520 this.logger.debug(_scope, 'rejected filename', { fileName });
521 return this.handlerNotFound(req, res, ctx);
522 }
523
524 const filePath = path.join(directory, fileName);
525
526 // File must exist, before any alternate static encodings will be considered.
527 let [stat, data] = await this._readFileInfo(filePath);
528 if (!stat) {
529 return this.handlerNotFound(req, res, ctx);
530 }
531
532 // If encodings were requested, check for static versions to serve.
533 // Update stat and data if matching version is found.
534 ctx.availableEncodings = Dingus.getResponseEncoding(Object.values(Enum.EncodingType), req);
535 if (ctx.availableEncodings.length === 0) {
536 // Identity encoding was specifically denied, and nothing else available.
537 this.logger.debug(_scope, 'no suitable encodings', { ctx });
538 return this.handlerMethodNotAllowed(req, res, ctx);
539 }
540 for (const encoding of ctx.availableEncodings) {
541 if (encoding === Enum.EncodingType.Identity) {
542 break;
543 }
544 const suffix = Enum.EncodingTypeSuffix[encoding];
545 if (suffix) {
546 const encodedFilePath = `${filePath}${suffix}`;
547 const [ encodedStat, encodedData ] = await this._readFileInfo(encodedFilePath);
548 if (encodedStat) {
549 ([ stat, data ] = [ encodedStat, encodedData ]);
550 ctx.selectedEncoding = encoding;
551 Dingus.addEncodingHeader(res, encoding);
552 res.setHeader(Enum.Header.Vary, Enum.Header.AcceptEncoding);
553 this.logger.debug(_scope, 'serving encoded version', { ctx, encodedFilePath });
554 }
555 break;
556 }
557 }
558
559 const lastModifiedDate = new Date(stat.mtimeMs);
560 res.setHeader(Enum.Header.LastModified, lastModifiedDate.toGMTString());
561
562 const eTag = common.generateETag(filePath, stat, data);
563 res.setHeader(Enum.Header.ETag, eTag);
564
565 if (common.isClientCached(req, stat.mtimeMs, eTag)) {
566 this.logger.debug(_scope, 'client cached file', { filePath });
567 res.statusCode = 304; // Not Modified
568 res.end();
569 return;
570 }
571
572 // Set the type based on extension of un-encoded filename.
573 const ext = path.extname(filePath).slice(1); // Drop the dot
574 const contentType = extensionToMime(ext);
575 res.setHeader(Enum.Header.ContentType, contentType);
576
577 // We presume static files are relatively cacheable.
578 res.setHeader(Enum.Header.CacheControl, 'public');
579
580 if (this.staticMetadata) {
581 await this._serveFileMetaHeaders(res, directory, fileName);
582 }
583
584 this.logger.debug(_scope, 'serving file', { filePath, contentType });
585 res.end(data);
586 }
587
588
589 /**
590 * Return a content-type appropriate rendering of an errorResponse object.
591 * @param {string} type content-type of response
592 * @param {object} err either an Error object, or an error response
593 * @param {number} err.statusCode
594 * @param {string} err.errorMessage
595 * @param {string|string[]} err.details
596 */
597 // eslint-disable-next-line class-methods-use-this
598 renderError(contentType, err) {
599 switch (contentType) {
600 case Enum.ContentType.ApplicationJson:
601 return JSON.stringify(err);
602
603 case Enum.ContentType.TextHTML:
604 return Template.errorHTML(err);
605
606 case Enum.ContentType.TextPlain:
607 default:
608 return [err.errorMessage, err.details].join('\r\n');
609 }
610 }
611
612
613 /**
614 * Send an error response. Terminal.
615 * Logs any non-error-response errors as such.
616 * @param {object} err either an Error object, or an error response
617 * @param {http.ClientRequest} req
618 * @param {http.ServerResponse} res
619 * @param {object} ctx
620 */
621 sendErrorResponse(err, req, res, ctx) {
622 const _scope = _fileScope('sendErrorResponse');
623 let body;
624
625 // Default to a content type if one is not yet present
626 if (!res.hasHeader(Enum.Header.ContentType)) {
627 res.setHeader(Enum.Header.ContentType, Enum.ContentType.TextPlain);
628 }
629
630 if (err && err.statusCode) {
631 res.statusCode = err.statusCode;
632 body = this.renderError(res.getHeader(Enum.Header.ContentType), err);
633 this.logger.debug(_scope, 'handler error', { err, ...common.handlerLogData(req, res, ctx) });
634 } else {
635 res.statusCode = 500;
636 body = this.renderError(res.getHeader(Enum.Header.ContentType), Enum.ErrorResponse.InternalServerError);
637 this.logger.error(_scope, 'handler exception', { err, ...common.handlerLogData(req, res, ctx) });
638 }
639 res.end(body);
640 }
641
642
643 /**
644 * @param {http.ClientRequest} req
645 * @param {http.ServerResponse} res
646 * @param {object} ctx
647 * @param {String} file - override ctx.params.file
648 */
649 async handlerGetStaticFile(req, res, ctx, file) {
650 Dingus.setHeadHandler(req, res, ctx);
651
652 // Set a default response type to handle any errors; will be re-set to serve actual static content type.
653 this.setResponseType(this.responseTypes, req, res, ctx);
654
655 await this.serveFile(req, res, ctx, this.staticPath, file || ctx.params.file);
656 }
657
658
659 /**
660 * @param {http.ClientRequest} req
661 * @param {http.ServerResponse} res
662 * @param {Object} ctx
663 * @param {String} newPath
664 * @param {Number} statusCode
665 */
666 async handlerRedirect(req, res, ctx, newPath, statusCode = 307) {
667 this.setResponseType(this.responseTypes, req, res, ctx);
668 res.setHeader(Enum.Header.Location, newPath);
669 res.statusCode = statusCode;
670 res.end();
671 }
672
673
674 /**
675 * @param {http.ClientRequest} req
676 * @param {http.ServerResponse} res
677 * @param {object} ctx
678 */
679 async handlerMethodNotAllowed(req, res, ctx) {
680 this.setResponseType(this.responseTypes, req, res, ctx);
681 throw new ResponseError(Enum.ErrorResponse.MethodNotAllowed);
682 }
683
684
685 /**
686 * @param {http.ClientRequest} req
687 * @param {http.ServerResponse} res
688 * @param {object} ctx
689 */
690 async handlerNotFound(req, res, ctx) {
691 this.setResponseType(this.responseTypes, req, res, ctx);
692 throw new ResponseError(Enum.ErrorResponse.NotFound);
693 }
694
695
696 /**
697 * @param {http.ClientRequest} req
698 * @param {http.ServerResponse} res
699 * @param {object} ctx
700 */
701 async handlerBadRequest(req, res, ctx) {
702 this.setResponseType(this.responseTypes, req, res, ctx);
703 throw new ResponseError(Enum.ErrorResponse.BadRequest);
704 }
705
706
707 /**
708 * @param {http.ClientRequest} req
709 * @param {http.ServerResponse} res
710 * @param {object} ctx
711 */
712 async handlerInternalServerError(req, res, ctx) {
713 this.setResponseType(this.responseTypes, req, res, ctx);
714 throw new ResponseError(Enum.ErrorResponse.InternalServerError);
715 }
716
717 }
718
719 module.exports = Dingus;