serveFile now requires a directory be specified
[squeep-api-dingus] / lib / dingus.js
1 /* eslint-disable security/detect-object-injection */
2 'use strict';
3
4 /**
5 * A very minimal API server framework.
6 * Just a self-contained router and some request glue.
7 */
8
9 require('./patches');
10 const { promises: fsPromises } = require('fs');
11 const path = require('path');
12 const querystring = require('querystring');
13 const common = require('./common');
14 const ContentNegotiation = require('./content-negotiation');
15 const Enum = require('./enum');
16 const { DingusError, ResponseError } = require('./errors');
17 const { extensionToMime } = require('./mime-helper');
18 const Router = require('./router');
19 const Template = require('./template');
20
21 // For logging.
22 const _fileScope = common.fileScope(__filename);
23
24 const defaultOptions = {
25 ignoreTrailingSlash: false,
26 proxyPrefix: '',
27 strictAccept: true,
28 selfBaseUrl: '',
29 staticMetadata: true,
30 staticPath: undefined, // no reasonable default
31 trustProxy: true,
32 querystring,
33 };
34
35 class Dingus {
36 /**
37 * @param {Object} logger object which implements logging methods
38 * @param {Object} options
39 * @param {Boolean} options.ignoreTrailingSlash
40 * @param {string} options.proxyPrefix leading part of url path to strip
41 * @param {Boolean} options.strictAccept whether to error on unsupported Accept type
42 * @param {string} options.selfBaseUrl for constructing links
43 * @param {Boolean} options.staticMetadata serve static headers with static files
44 * @param {Boolean} options.trustProxy trust some header data to be provided by proxy
45 * @param {Object} options.querystring alternate qs parser to use
46 */
47 constructor(logger = common.nullLogger, options = {}) {
48 common.setOptions(this, defaultOptions, options);
49
50 this.router = new Router(options);
51
52 if (!this.proxyPrefix) {
53 this._stripPrefix = (p) => p;
54 }
55
56 this.responseTypes = [
57 Enum.ContentType.TextHTML,
58 Enum.ContentType.TextPlain,
59 Enum.ContentType.ApplicationJson,
60 ];
61
62 this.logger = logger;
63 common.ensureLoggerLevels(this.logger);
64 }
65
66
67 /**
68 * Resolve relative and empty paths in url
69 * @param {string} p path
70 */
71 _normalizePath(p) {
72 const pathNorm = path.normalize(p); // This isn't perfectly correct, but it's easy...
73 return this._stripPrefix(pathNorm);
74 }
75
76
77 /**
78 * Remove a leading portion of url path
79 * N.B. This method gets obliterated if there is no prefix defined at construction
80 * @param {string} p path
81 */
82 _stripPrefix(p) {
83 if (p.startsWith(this.proxyPrefix)) {
84 return p.slice(this.proxyPrefix.length);
85 }
86 return p;
87 }
88
89
90 /**
91 * Returns the path part, and querystring object, from a request url.
92 * @param {string} url
93 */
94 _splitUrl(url) {
95 const [ p, qs ] = common.splitFirst(url, '?');
96 return {
97 pathPart: this._normalizePath(p),
98 queryParams: this.querystring.parse(qs),
99 };
100 }
101
102
103 /**
104 * Insert a new path handler
105 * @param {string} method
106 * @param {string} urlPath
107 * @param {fn} handler
108 */
109 on(method, urlPath, handler, ...handlerArgs) {
110 this.router.on(method, urlPath, handler, handlerArgs);
111 }
112
113
114 /**
115 * Common header tagging for all requests.
116 * Add our own identifier, and persist any external transit identifiers.
117 * @param {http.ClientRequest} req
118 * @param {http.ServerResponse} res
119 * @param {object} ctx
120 */
121 static tagContext(req, res, ctx) {
122 const requestId = common.requestId();
123 ctx.requestId = requestId;
124 res.setHeader(Enum.Header.RequestId, requestId);
125 [Enum.Header.XRequestId, Enum.Header.XCorrelationId].forEach((h) => {
126 const v = req.getHeader(h);
127 if (v) {
128 ctx[h.replace(/-/g, '')] = v;
129 res.setHeader(h, v);
130 }
131 });
132 return requestId;
133 }
134
135
136 /**
137 *
138 * @param {http.ClientRequest} req
139 */
140 _getAddress(req) {
141 // TODO: RFC7239 Forwarded support
142 const address = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedFor)) ||
143 (this.trustProxy && req && req.getHeader(Enum.Header.XRealIP)) ||
144 (req && req.connection && req.connection.remoteAddress) ||
145 '';
146 return address.split(/\s*,\s*/u)[0];
147 }
148
149
150 /**
151 *
152 * @param {http.ClientRequest} req
153 */
154 _getProtocol(req) {
155 // TODO: RFC7239 Forwarded support
156 const protocol = (this.trustProxy && req && req.getHeader(Enum.Header.XForwardedProto)) ||
157 ((req && req.connection && req.connection.encrypted) ? 'https' : 'http');
158 return protocol.split(/\s*,\s*/u)[0];
159 }
160
161
162 /**
163 *
164 * @param {http.ClientRequest} req
165 * @param {http.ServerResponse} res
166 * @param {object} ctx
167 */
168 clientAddressContext(req, res, ctx) {
169 ctx.clientAddress = this._getAddress(req);
170 ctx.clientProtocol = this._getProtocol(req);
171 }
172
173
174 /**
175 * Called before every request handler.
176 * @param {http.ClientRequest} req
177 * @param {http.ServerResponse} res
178 * @param {object} ctx
179 */
180 async preHandler(req, res, ctx) {
181 Dingus.tagContext(req, res, ctx);
182 this.clientAddressContext(req, res, ctx);
183 }
184
185
186 /**
187 * Helper for collecting chunks as array of buffers.
188 * @param {Buffer[]} chunks
189 * @param {string|Buffer} chunk
190 * @param {string} encoding
191 */
192 static pushBufChunk(chunks, chunk, encoding = 'utf8') {
193 if (chunk) {
194 if (typeof chunk === 'string') {
195 chunk = Buffer.from(chunk, encoding);
196 }
197 chunks.push(chunk);
198 }
199 }
200
201
202 /**
203 * Sets ctx.responseBody and calls handler upon res.end().
204 * @param {http.ClientRequest} req
205 * @param {http.ServerResponse} res
206 * @param {object} ctx
207 * @param {*} handler fn(req, res, ctx)
208 */
209 static setEndBodyHandler(req, res, ctx, handler) {
210 const origWrite = res.write.bind(res);
211 const origEnd = res.end.bind(res);
212 const chunks = [];
213 res.write = function (chunk, encoding, ...rest) {
214 Dingus.pushBufChunk(chunks, chunk, encoding);
215 return origWrite(chunk, encoding, ...rest);
216 };
217 res.end = function (data, encoding, ...rest) {
218 Dingus.pushBufChunk(chunks, data, encoding);
219 ctx.responseBody = Buffer.concat(chunks);
220 handler(req, res, ctx);
221 return origEnd(data, encoding, ...rest);
222 };
223 }
224
225
226 /**
227 * Intercept writes for head requests, do not send to client,
228 * but send length, and make body available in context.
229 * @param {http.ClientRequest} req
230 * @param {http.ServerResponse} res
231 * @param {object} ctx
232 */
233 static setHeadHandler(req, res, ctx) {
234 if (req.method === 'HEAD') {
235 const origEnd = res.end.bind(res);
236 const chunks = [];
237 res.write = function (chunk, encoding) {
238 Dingus.pushBufChunk(chunks, chunk, encoding);
239 // No call to original res.write.
240 };
241 res.end = function (data, encoding, ...rest) {
242 Dingus.pushBufChunk(chunks, data, encoding);
243 ctx.responseBody = Buffer.concat(chunks);
244 res.setHeader(Enum.Header.ContentLength, Buffer.byteLength(ctx.responseBody));
245 return origEnd(undefined, encoding, ...rest);
246 };
247 }
248 }
249
250
251 /**
252 * Dispatch the handler for a request
253 * @param {http.ClientRequest} req
254 * @param {http.ServerResponse} res
255 * @param {object} ctx
256 */
257 async dispatch(req, res, ctx = {}) {
258 const _scope = _fileScope('dispatch');
259
260 const { pathPart, queryParams } = this._splitUrl(req.url);
261 ctx.queryParams = queryParams;
262
263 let handler, handlerArgs = [];
264 try {
265 ({ handler, handlerArgs } = this.router.lookup(req.method, pathPart, ctx));
266 } catch (e) {
267 if (e instanceof DingusError) {
268 switch (e.message) {
269 case 'NoPath':
270 handler = this.handlerNotFound.bind(this);
271 break;
272 case 'NoMethod':
273 handler = this.handlerMethodNotAllowed.bind(this);
274 break;
275 default:
276 this.logger.error(_scope, 'unknown dingus error', { error: e });
277 handler = this.handlerInternalServerError.bind(this);
278 }
279 } else if (e instanceof URIError) {
280 handler = this.handlerBadRequest.bind(this);
281 } else {
282 this.logger.error(_scope, 'lookup failure', { error: e });
283 handler = this.handlerInternalServerError.bind(this);
284 }
285 }
286
287 try {
288 await this.preHandler(req, res, ctx);
289 return await handler(req, res, ctx, ...handlerArgs);
290 } catch (e) {
291 ctx.error = e;
292 this.sendErrorResponse(e, req, res, ctx);
293 }
294 }
295
296
297 /**
298 * Return normalized type, without any parameters.
299 * @param {http.ClientRequest} req
300 * @returns {string}
301 */
302 static getRequestContentType(req) {
303 const contentType = req.getHeader(Enum.Header.ContentType);
304 return (contentType || '').split(';')[0].trim().toLowerCase();
305 }
306
307
308 /**
309 * Parse rawBody from ctx as contentType into parsedBody.
310 * @param {string} contentType
311 * @param {object} ctx
312 */
313 parseBody(contentType, ctx) {
314 const _scope = _fileScope('parseBody');
315
316 switch (contentType) {
317 case Enum.ContentType.ApplicationForm:
318 ctx.parsedBody = this.querystring.parse(ctx.rawBody);
319 break;
320
321 case Enum.ContentType.ApplicationJson:
322 try {
323 ctx.parsedBody = JSON.parse(ctx.rawBody);
324 } catch (e) {
325 this.logger.debug(_scope, 'JSON parse failed', { requestId: ctx.requestId, error: e });
326 throw new ResponseError(Enum.ErrorResponse.BadRequest, e.message);
327 }
328 break;
329
330 default:
331 this.logger.debug(_scope, 'unhandled content-type', { requestId: ctx.requestId, contentType });
332 throw new ResponseError(Enum.ErrorResponse.UnsupportedMediaType);
333 }
334 }
335
336
337 /**
338 * Return all body data from a request.
339 * @param {http.ClientRequest} req
340 */
341 async bodyData(req) {
342 const _scope = _fileScope('bodyData');
343 return new Promise((resolve, reject) => {
344 const body = [];
345 req.on('data', (chunk) => body.push(chunk));
346 req.on('end', () => resolve(Buffer.concat(body).toString()));
347 req.on('error', (e) => {
348 this.logger.error(_scope, 'failed', { error: e });
349 reject(e);
350 });
351 });
352 }
353
354
355 /**
356 * Read and parse request body data.
357 * @param {http.ClientRequest} req
358 * @param {http.ServerResponse} res
359 * @param {object} ctx
360 */
361 async ingestBody(req, res, ctx) {
362 ctx.rawBody = await this.bodyData(req);
363 const contentType = Dingus.getRequestContentType(req);
364 this.parseBody(contentType, ctx);
365 }
366
367
368 /**
369 * Return the best matching response type.
370 * @param {string[]} responseTypes
371 * @param {http.ClientRequest} req
372 */
373 static getResponseContentType(responseTypes, req) {
374 const acceptHeader = req.getHeader(Enum.Header.Accept);
375 return ContentNegotiation.accept(responseTypes, acceptHeader);
376 }
377
378
379 /**
380 * Returns a list of the most-preferred content encodings for the response.
381 * @param {string[]} responseEncodings
382 * @param {http.ClientRequest} req
383 */
384 static getResponseEncoding(responseEncodings, req) {
385 const acceptEncodingHeader = req.getHeader(Enum.Header.AcceptEncoding);
386 return ContentNegotiation.preferred(responseEncodings, acceptEncodingHeader);
387 }
388
389
390 /**
391 * Set the best content type for the response.
392 * @param {string[]} responseTypes default first
393 * @param {http.ClientRequest} req
394 * @param {http.ServerResponse} res
395 * @param {object} ctx
396 */
397 setResponseType(responseTypes, req, res, ctx) {
398 const _scope = _fileScope('setResponseType');
399 ctx.responseType = Dingus.getResponseContentType(responseTypes, req);
400 if (!ctx.responseType) {
401 if (this.strictAccept) {
402 this.logger.debug(_scope, 'unhandled strict accept', { requestId: req.requestId });
403 throw new ResponseError(Enum.ErrorResponse.NotAcceptable);
404 } else {
405 ctx.responseType = responseTypes[0];
406 }
407 }
408 res.setHeader(Enum.Header.ContentType, ctx.responseType);
409 }
410
411
412 /**
413 * Inserts an encoding
414 * @param {http.ServerResponse} res
415 * @param {string} encoding
416 */
417 static addEncodingHeader(res, encoding) {
418 const existingEncodings = res.getHeader(Enum.Header.ContentEncoding);
419 if (existingEncodings) {
420 encoding = `${encoding}, ${existingEncodings}`;
421 }
422 res.setHeader(Enum.Header.ContentEncoding, encoding);
423 }
424
425
426 /**
427 * Attempt to fetch both data and metadata for a file.
428 * @param {string} filePath
429 */
430 async _readFileInfo(filePath) {
431 const _scope = _fileScope('_readFileInfo');
432 let result;
433 try {
434 // eslint-disable-next-line security/detect-non-literal-fs-filename
435 const stat = fsPromises.stat(filePath);
436 // eslint-disable-next-line security/detect-non-literal-fs-filename
437 const data = fsPromises.readFile(filePath);
438 result = await Promise.all([stat, data]);
439 } catch (e) {
440 if (['ENOENT', 'EACCES', 'EISDIR', 'ENAMETOOLONG', 'EINVAL'].includes(e.code)) {
441 return [null, null];
442 }
443 this.logger.error(_scope, 'fs error', { error: e, filePath });
444 throw e;
445 }
446 return result;
447 }
448
449
450 /**
451 * Potentially add additional headers from static file meta-file.
452 * @param {http.ServerResponse} res
453 * @param {string} directory
454 * @param {string} fileName - already normalized and filtered
455 */
456 async _serveFileMetaHeaders(res, directory, fileName) {
457 const _scope = _fileScope('_serveFileMetaHeaders');
458 this.logger.debug(_scope, 'called', { directory, fileName });
459
460 const metaPrefix = '.';
461 const metaSuffix = '.meta';
462 const metaFileName = `${metaPrefix}${fileName}${metaSuffix}`;
463 const metaFilePath = path.join(directory, metaFileName);
464
465 const [stat, data] = await this._readFileInfo(metaFilePath);
466 if (!stat) {
467 return;
468 }
469
470 const lineBreakRE = /\r\n|\n|\r/;
471 const lines = data.toString().split(lineBreakRE);
472 common.unfoldHeaderLines(lines);
473
474 const headerParseRE = /^(?<name>[^:]+): +(?<value>.*)$/;
475 lines.forEach((line) => {
476 if (line) {
477 const result = headerParseRE.exec(line);
478 const { groups: header } = result;
479 res.setHeader(header.name, header.value);
480 }
481 });
482 }
483
484
485 /**
486 * Serve a file from a directory, with rudimentary cache awareness.
487 * This will also serve pre-encoded variations if available and requested.
488 * @param {http.ClientRequest} req
489 * @param {http.ServerResponse} res
490 * @param {object} ctx
491 * @param {string} directory
492 * @param {string} fileName
493 */
494 async serveFile(req, res, ctx, directory, fileName) {
495 const _scope = _fileScope('serveFile');
496 this.logger.debug(_scope, 'called', { req: common.requestLogData(req), ctx });
497
498 // Require a directory field.
499 if (!directory) {
500 this.logger.debug(_scope, 'rejected unset directory', { fileName });
501 return this.handlerNotFound(req, res, ctx);
502 }
503
504 // Normalize the supplied path, as encoded path-navigation may have been (maliciously) present.
505 fileName = path.normalize(fileName);
506
507 // We will not deal with any subdirs, nor any dot-files.
508 // (Note that we could not deal with subdirs even if we wanted, due to simple router matching scheme.)
509 if (fileName.indexOf(path.sep) >= 0
510 || fileName.charAt(0) === '.') {
511 this.logger.debug(_scope, 'rejected filename', { fileName });
512 return this.handlerNotFound(req, res, ctx);
513 }
514
515 const filePath = path.join(directory, fileName);
516
517 // File must exist, before any alternate static encodings will be considered.
518 let [stat, data] = await this._readFileInfo(filePath);
519 if (!stat) {
520 return this.handlerNotFound(req, res, ctx);
521 }
522
523 // If encodings were requested, check for static versions to serve.
524 // Update stat and data if matching version is found.
525 ctx.availableEncodings = Dingus.getResponseEncoding(Object.values(Enum.EncodingType), req);
526 if (ctx.availableEncodings.length === 0) {
527 // Identity encoding was specifically denied, and nothing else available.
528 this.logger.debug(_scope, 'no suitable encodings', { ctx });
529 return this.handlerMethodNotAllowed(req, res, ctx);
530 }
531 for (const encoding of ctx.availableEncodings) {
532 if (encoding === Enum.EncodingType.Identity) {
533 break;
534 }
535 const suffix = Enum.EncodingTypeSuffix[encoding];
536 if (suffix) {
537 const encodedFilePath = `${filePath}${suffix}`;
538 const [ encodedStat, encodedData ] = await this._readFileInfo(encodedFilePath);
539 if (encodedStat) {
540 ([ stat, data ] = [ encodedStat, encodedData ]);
541 ctx.selectedEncoding = encoding;
542 Dingus.addEncodingHeader(res, encoding);
543 res.setHeader(Enum.Header.Vary, Enum.Header.AcceptEncoding);
544 this.logger.debug(_scope, 'serving encoded version', { ctx, encodedFilePath });
545 }
546 break;
547 }
548 }
549
550 const lastModifiedDate = new Date(stat.mtimeMs);
551 res.setHeader(Enum.Header.LastModified, lastModifiedDate.toGMTString());
552
553 const eTag = common.generateETag(filePath, stat, data);
554 res.setHeader(Enum.Header.ETag, eTag);
555
556 if (common.isClientCached(req, stat.mtimeMs, eTag)) {
557 this.logger.debug(_scope, 'client cached file', { filePath });
558 res.statusCode = 304; // Not Modified
559 res.end();
560 return;
561 }
562
563 // Set the type based on extension of un-encoded filename.
564 const ext = path.extname(filePath).slice(1); // Drop the dot
565 const contentType = extensionToMime(ext);
566 res.setHeader(Enum.Header.ContentType, contentType);
567
568 // We presume static files are relatively cacheable.
569 res.setHeader(Enum.Header.CacheControl, 'public');
570
571 if (this.staticMetadata) {
572 await this._serveFileMetaHeaders(res, directory, fileName);
573 }
574
575 this.logger.debug(_scope, 'serving file', { filePath, contentType });
576 res.end(data);
577 }
578
579
580 /**
581 * Return a content-type appropriate rendering of an errorResponse object.
582 * @param {string} type content-type of response
583 * @param {object} err either an Error object, or an error response
584 * @param {number} err.statusCode
585 * @param {string} err.errorMessage
586 * @param {string|string[]} err.details
587 */
588 // eslint-disable-next-line class-methods-use-this
589 renderError(contentType, err) {
590 switch (contentType) {
591 case Enum.ContentType.ApplicationJson:
592 return JSON.stringify(err);
593
594 case Enum.ContentType.TextHTML:
595 return Template.errorHTML(err);
596
597 case Enum.ContentType.TextPlain:
598 default:
599 return [err.errorMessage, err.details].join('\r\n');
600 }
601 }
602
603
604 /**
605 * Send an error response. Terminal.
606 * Logs any non-error-response errors as such.
607 * @param {object} err either an Error object, or an error response
608 * @param {http.ClientRequest} req
609 * @param {http.ServerResponse} res
610 * @param {object} ctx
611 */
612 sendErrorResponse(err, req, res, ctx) {
613 const _scope = _fileScope('sendErrorResponse');
614 let body;
615
616 // Default to a content type if one is not yet present
617 if (!res.hasHeader(Enum.Header.ContentType)) {
618 res.setHeader(Enum.Header.ContentType, Enum.ContentType.TextPlain);
619 }
620
621 if (err && err.statusCode) {
622 res.statusCode = err.statusCode;
623 body = this.renderError(res.getHeader(Enum.Header.ContentType), err);
624 this.logger.debug(_scope, 'handler error', { err, ...common.handlerLogData(req, res, ctx) });
625 } else {
626 res.statusCode = 500;
627 body = this.renderError(res.getHeader(Enum.Header.ContentType), Enum.ErrorResponse.InternalServerError);
628 this.logger.error(_scope, 'handler exception', { err, ...common.handlerLogData(req, res, ctx) });
629 }
630 res.end(body);
631 }
632
633
634 /**
635 * @param {http.ClientRequest} req
636 * @param {http.ServerResponse} res
637 * @param {object} ctx
638 * @param {String} file - override ctx.params.file
639 */
640 async handlerGetStaticFile(req, res, ctx, file) {
641 Dingus.setHeadHandler(req, res, ctx);
642
643 // Set a default response type to handle any errors; will be re-set to serve actual static content type.
644 this.setResponseType(this.responseTypes, req, res, ctx);
645
646 await this.serveFile(req, res, ctx, this.staticPath, file || ctx.params.file);
647 }
648
649
650 /**
651 * @param {http.ClientRequest} req
652 * @param {http.ServerResponse} res
653 * @param {Object} ctx
654 * @param {String} newPath
655 * @param {Number} statusCode
656 */
657 async handlerRedirect(req, res, ctx, newPath, statusCode = 307) {
658 this.setResponseType(this.responseTypes, req, res, ctx);
659 res.setHeader(Enum.Header.Location, newPath);
660 res.statusCode = statusCode;
661 res.end();
662 }
663
664
665 /**
666 * @param {http.ClientRequest} req
667 * @param {http.ServerResponse} res
668 * @param {object} ctx
669 */
670 async handlerMethodNotAllowed(req, res, ctx) {
671 this.setResponseType(this.responseTypes, req, res, ctx);
672 throw new ResponseError(Enum.ErrorResponse.MethodNotAllowed);
673 }
674
675
676 /**
677 * @param {http.ClientRequest} req
678 * @param {http.ServerResponse} res
679 * @param {object} ctx
680 */
681 async handlerNotFound(req, res, ctx) {
682 this.setResponseType(this.responseTypes, req, res, ctx);
683 throw new ResponseError(Enum.ErrorResponse.NotFound);
684 }
685
686
687 /**
688 * @param {http.ClientRequest} req
689 * @param {http.ServerResponse} res
690 * @param {object} ctx
691 */
692 async handlerBadRequest(req, res, ctx) {
693 this.setResponseType(this.responseTypes, req, res, ctx);
694 throw new ResponseError(Enum.ErrorResponse.BadRequest);
695 }
696
697
698 /**
699 * @param {http.ClientRequest} req
700 * @param {http.ServerResponse} res
701 * @param {object} ctx
702 */
703 async handlerInternalServerError(req, res, ctx) {
704 this.setResponseType(this.responseTypes, req, res, ctx);
705 throw new ResponseError(Enum.ErrorResponse.InternalServerError);
706 }
707
708 }
709
710 module.exports = Dingus;