X-Git-Url: http://git.squeep.com/?a=blobdiff_plain;f=lib%2Fmystery-box.js;h=a3e3192047a30d80a89a597e0158e46eb23b3f9e;hb=HEAD;hp=6198cc816484b406804cf4a4cdd3c84685bce60e;hpb=383899395bf599359d60337edb651af0592f82cb;p=squeep-mystery-box diff --git a/lib/mystery-box.js b/lib/mystery-box.js index 6198cc8..a3e3192 100644 --- a/lib/mystery-box.js +++ b/lib/mystery-box.js @@ -1,19 +1,25 @@ 'use strict'; +const { EventEmitter } = require('events'); const crypto = require('crypto'); const zlib = require('zlib'); const { promisify } = require('util'); -const common = require('./common'); +const { MysteryBoxError } = require('./errors'); const allVersions = require('./version-parameters'); const { performance } = require('perf_hooks'); +const { name: packageName, version: packageVersion } = require('../package.json'); -const _fileScope = common.fileScope(__filename); +const packageInfo = { + packageName, + packageVersion, +}; const brotliCompressAsync = promisify(zlib.brotliCompress); const brotliDecompressAsync = promisify(zlib.brotliDecompress); const deflateRawAsync = promisify(zlib.deflateRaw); const inflateRawAsync = promisify(zlib.inflateRaw); const scryptAsync = promisify(crypto.scrypt); +const randomBytesAsync = promisify(crypto.randomBytes); /** * Only you will know what's inside your... @@ -57,28 +63,29 @@ const compressionFlagsShift = 0; const payloadFlagsMask = (availableFlags.BufferPayload); const payloadFlagsShift = 7; - -class MysteryBox { +class MysteryBox extends EventEmitter { /** - * @param {Console} logger - * @param {Object} options - * @param {String} options.encryptionSecret - * @param {Number=} options.defaultFlags + * @param {object} options options + * @param {string|string[]} options.encryptionSecret if an array, will always encrypt with first secret, will attempt to decrypt with all; useful for rolling secrets + * @param {number=} options.defaultFlags default flags to use when packing boxes + * @param {any[]} args passed on to EventEmitter constructor */ - constructor(logger, options = {}) { - this.logger = logger; - this.options = options; - - this.secret = options.encryptionSecret; - if (!this.secret) { - throw new Error('missing encryption secret'); + constructor(options = {}, ...args) { + super(...args); + this.secrets = MysteryBox._ensureArray(options.encryptionSecret); + if (!this.secrets.length) { + throw new MysteryBoxError('missing encryption secret'); } - // TODO: support secret rolling // Filter any unavailable algorithms const availableCiphers = crypto.getCiphers(); + const availableHashes = crypto.getHashes(); + // Add legacy scrypt to available hashes for filtering key derivations + availableHashes.push(allVersions.KD.SCRYPT); this.versionParameters = Object.entries(allVersions).reduce((acc, [v, p]) => { - if (availableCiphers.includes(p.algorithm)) { + const validCipher = availableCiphers.includes(p.algorithm); + const validKeyDeriver = availableHashes.includes(p.keyDeriver); + if (validCipher && validKeyDeriver) { acc[v] = p; // eslint-disable-line security/detect-object-injection } return acc; @@ -87,19 +94,37 @@ class MysteryBox { // Default to highest this.bestVersion = Number(Object.keys(this.versionParameters).sort().pop()); if (Number.isNaN(this.bestVersion)) { - throw new Error('no supported versions available'); + throw new MysteryBoxError('no supported versions available'); } this.Flags = availableFlags; this.defaultFlags = 'defaultFlags' in options ? options.defaultFlags : availableFlags.Flate; if (this.defaultFlags < 0 || this.defaultFlags > 255) { - throw new RangeError('Invalid default flag value'); + throw new MysteryBoxError('Invalid default flag value'); + } + } + + + /** + * Return an array containing x if x is something and not an array + * @param {*} x thing to ensure is an array + * @returns {any[]} thing array or thing in an array + */ + static _ensureArray(x) { + if (x === undefined) { + return []; + } + if (!Array.isArray(x)) { + return Array(x); } + return x; } /** * Parse the bits out of the flags. + * @param {number} flags flags + * @returns {object} decoded flags */ static _decodeFlags(flags) { return { @@ -109,43 +134,224 @@ class MysteryBox { } + /** + * Generate key data. + * @param {string} deriver key deriver enum + * @param {Buffer} secret secret + * @param {Buffer} salt salt + * @param {number} keyBytes bytes in key + * @returns {Promise} key + */ + static async _keyFromSecret(deriver, secret, salt, keyBytes) { + switch (deriver) { + case allVersions.KD.SHAKE256: { + const hash = crypto.createHash(allVersions.KD.SHAKE256, { outputLength: keyBytes }); + hash.update(salt); + hash.update(secret); + return hash.digest(); + } + + case allVersions.KD.BLAKE2B512: { + const hash = crypto.createHash(allVersions.KD.BLAKE2B512); + hash.update(salt); + hash.update(secret); + const digest = hash.digest(); + // should assert that keyBytes <= 64 + // but not concerned about that for now + // until we have a new algorithm with bigger key size + return digest.subarray(0, keyBytes); + } + + case allVersions.KD.SCRYPT: + return scryptAsync(secret, salt, keyBytes); + + default: + throw new MysteryBoxError('unsupported key deriver'); + } + } + + + /** + * Return bits and bit mask for given number of encoded bytes. + * @param {number} numBytes number of bytes + * @returns {object} bits value and mask + */ + static _versionHeaderBits(numBytes) { + // Round up to 8 bits in result, just to be proper. + const resultBits = (((numBytes + 7) >> 3) << 3) >>> 0; + return { + headerValue: ((0xff << (resultBits - numBytes + 1)) & 0xff) >>> 0, + headerMask: ((0xff << (resultBits - numBytes)) & 0xff) >>> 0, + }; + } + + + /** + * Parse a byte into the total number of bytes in the packed number, + * returning that and the new value for the first byte, to update in the + * buffer before parsing as an unsigned integer. + * Number of packed bytes is indicated by location of first leading 0 bit. + * Support for numbers larger than 127 is of dubious practicality, but here + * it is anyhow. + * @param {number} firstByte first byte contains encoded byte length + * @returns {object} number of bytes and masked first byte + */ + static _versionHeaderDecode(firstByte) { + for (let numBytes = 1; numBytes <= 8; numBytes++) { + const { + headerValue, + headerMask, + } = MysteryBox._versionHeaderBits(numBytes); + if (((firstByte & headerMask) >>> 0) === headerValue) { + const restMask = (~headerMask & 0xff) >>> 0; + return { + numBytes, + firstByte: (firstByte & restMask) >>> 0, + }; + } + } + // Nine bytes would be an extravagence. + throw new MysteryBoxError(`unsupported version header (0x${firstByte.toString(16)})`); + } + + + /** + * Decode leading bytes of buffer as version identifier. + * In the first byte, the position of the first unset bit indicates how + * many total bytes comprise the version, in big-endian encoding. + * Returns decoded version and number of bytes used to decode. + * Only supports up to 6-byte numbers, and of those, only up to 4398046511103. + * @param {Buffer} buf N.B. will be mogrified + * @returns {object} decoded version and version byte length + */ + static _versionDecode(buf) { + const headerByte = buf.readUInt8(0); + const { numBytes: versionBytes, firstByte } = MysteryBox._versionHeaderDecode(headerByte); + if (versionBytes === 1) { + return { + version: firstByte, + versionBytes, + }; + } + + if (versionBytes > 6) { + throw new MysteryBoxError(`unsupported version (${versionBytes} bytes)`); + } + + // Otherwise, update the masked first byte and parse the rest of the buffer. + buf[0] = firstByte; + return { + version: buf.readUIntBE(0, versionBytes), + versionBytes, + }; + } + + + /** + * Encode a version identifier into a buffer of a variable number of bytes. + * @param {number} version version number to encode + * @returns {object} encoded version buffer and number of bytes + */ + static _versionEncode(version) { + let versionBytes = 0; + + if (version <= 0x7f) { // 0-127 + versionBytes = 1; + } else if (version <= 0x3fff) { // 128-16383 + versionBytes = 2; + } else if (version <= 0x1fffff) { // 16384-2097151 + versionBytes = 3; + } else if (version <= 0x0fffffff) { // 2097152-268435455 + versionBytes = 4; + } else if (version <= 0x07ffffffff) { // 268435456-34359738367 + versionBytes = 5; + } else if (version <= 0x03ffffffffff) { // 34359738368-4398046511103 + versionBytes = 6; + } else { + throw new MysteryBoxError(`version too large to encode (${version})`); + } + + const buffer = Buffer.alloc(versionBytes); + buffer.writeUIntBE(version, 0, versionBytes); + const headerByte = ((0xff << (8 - versionBytes + 1)) & 0xff) >>> 0; + buffer[0] = (buffer[0] | headerByte) >>> 0; + + return { + buffer, + versionBytes, + }; + } + + + /** + * Stats tracked when packing/unpacking boxes. + * @param {string} method method name generating stats + * @returns {object} stats and timings + */ + static _newStats(method) { + return { + stats: { + method, + version: undefined, + flags: undefined, + flagsRaw: undefined, + serializedBytes: undefined, + compressedBytes: undefined, + }, + timingsMs: { + start: performance.now(), + preCompress: 0, + postCompress: 0, + preCrypt: 0, + postCrypt: 0, + end: 0, + }, + }; + } + + /** * Put contents into a mysterious box. - * @param {Object|Buffer} contents - * @returns {String} + * @param {object|Buffer} contents object or buffer + * @param {number=} version version + * @param {number=} flags flags + * @returns {Promise} encoded contents */ async pack(contents, version = this.bestVersion, flags = this.defaultFlags) { - const _scope = _fileScope('pack'); - const timingsMs = { - start: performance.now(), - preCompress: 0, - postCompress: 0, - preEncrypt: 0, - postEncrypt: 0, - end: 0, - }; + const { stats, timingsMs } = MysteryBox._newStats('pack'); if (!(version in this.versionParameters)) { - throw new RangeError(`MysteryBox format version ${version} not supported`); + throw new MysteryBoxError(`MysteryBox format version ${version} not supported`); } // eslint-disable-next-line security/detect-object-injection const v = this.versionParameters[version]; + stats.version = version; - const { compression } = MysteryBox._decodeFlags(flags); + const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags); if (Buffer.isBuffer(contents)) { + // Ensure payloadIsBuffer flag is set when contents are indeed a Buffer flags |= this.Flags.BufferPayload; } else { - contents = JSON.stringify(contents); + if (payloadIsBuffer) { + // Flag is set, but contents are not a Buffer? Try to coerce contents. + contents = Buffer.from(contents); + } else { + // Otherwise attempt to serialize the object + contents = JSON.stringify(contents); + } } + stats.serializedBytes = Buffer.byteLength(contents); - const versionBuffer = Buffer.alloc(v.versionBytes); - versionBuffer.writeUInt8(v.version, 0); + const { buffer: versionBuffer, versionBytes } = MysteryBox._versionEncode(v.version); + if (versionBytes !== v.versionBytes) { + throw new MysteryBoxError('internal inconsistency, mismatched version byte length'); + } const [iv, salt] = await Promise.all([ v.ivBytes, v.saltBytes, - ].map((b) => common.randomBytesAsync(b))); + ].map((b) => randomBytesAsync(b))); timingsMs.preCompress = performance.now(); let compressedContents; @@ -160,33 +366,40 @@ class MysteryBox { compressedContents = await deflateRawAsync(contents); break; } + stats.compressedBytes = Buffer.byteLength(compressedContents); timingsMs.postCompress = timingsMs.preCrypt = performance.now(); let payload; - if (compressedContents.length >= contents.length) { + if (stats.compressedBytes >= stats.serializedBytes) { + // If compression is not beneficial enough, or detrimental, do not use flags = flags & ~compressionFlagsMask; payload = contents; + stats.compressedBytes = undefined; } else { payload = compressedContents; } + const flagsBuffer = Buffer.alloc(v.flagsBytes); flagsBuffer.writeUInt8(flags, 0); + stats.flagsRaw = flags; + stats.flags = this._prettyFlags(flags); // Authenticate all this data const aadBuffer = Buffer.concat([versionBuffer, flagsBuffer, iv, salt]); - const key = await scryptAsync(this.secret, salt, v.keyBytes); + // Always encrypt with first secret + const secret = this.secrets[0]; + const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes); const cipher = crypto.createCipheriv(v.algorithm, key, iv, v.algOptions); cipher.setAAD(aadBuffer); const encrypted = cipher.update(payload); const final = cipher.final(); const tag = cipher.getAuthTag(); - const merged = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64'); - const result = common.base64ToBase64URL(merged); + const result = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64url'); timingsMs.end = timingsMs.postCrypt = performance.now(); - this.logger.debug(_scope, 'statistics', { version, flags: this._prettyFlags(flags), serialized: contents.length, compressed: payload.length, encoded: result.length, ...MysteryBox._timingsLog(timingsMs) }); + this.emit('statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs), ...packageInfo }); return result; } @@ -194,65 +407,79 @@ class MysteryBox { /** * Take contents out of a mysterious box. - * @param {String} box - Base64URL encoded payload - * @returns {Object} + * @param {string} box - Base64URL encoded payload + * @returns {Promise} decoded contents */ async unpack(box) { - const _scope = _fileScope('unpack'); - const timingsMs = { - start: performance.now(), - preCompress: 0, - postCompress: 0, - preCrypt: 0, - postCrypt: 0, - end: 0, - }; + const { stats, timingsMs } = MysteryBox._newStats('unpack'); if (!box) { - throw new RangeError('nothing to unpack'); + throw new MysteryBoxError('nothing to unpack'); } - const raw = Buffer.from(common.base64URLToBase64(box), 'base64'); + const raw = Buffer.from(box, 'base64url'); let offset = 0; - const version = raw.slice(offset, 1).readUInt8(0); + const { version, versionBytes } = MysteryBox._versionDecode(raw); if (!(version in this.versionParameters)) { - throw new RangeError('unsupported version'); + throw new MysteryBoxError(`unsupported version (${version})`); } // eslint-disable-next-line security/detect-object-injection const v = this.versionParameters[version]; + + if (v.versionBytes !== versionBytes) { + throw new MysteryBoxError('internal inconsistency, mismatched version byte length'); + } offset += v.versionBytes; + stats.version = version; const minBytes = v.versionBytes + v.flagsBytes + v.ivBytes + v.saltBytes + v.tagBytes; if (raw.length < minBytes) { - throw new RangeError('not enough to unpack'); + throw new MysteryBoxError('not enough to unpack'); } - const flags = raw.slice(offset, offset + v.flagsBytes).readUInt8(0); + const flags = raw.subarray(offset, offset + v.flagsBytes).readUInt8(0); offset += v.flagsBytes; + stats.flagsRaw = flags; const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags); - const iv = raw.slice(offset, offset + v.ivBytes); + const iv = raw.subarray(offset, offset + v.ivBytes); offset += v.ivBytes; - const salt = raw.slice(offset, offset + v.saltBytes); + const salt = raw.subarray(offset, offset + v.saltBytes); offset += v.saltBytes; - const aad = raw.slice(0, offset); // Everything up to here + const aad = raw.subarray(0, offset); // Everything up to here - const tag = raw.slice(offset, offset + v.tagBytes); + const tag = raw.subarray(offset, offset + v.tagBytes); offset += v.tagBytes; - const encrypted = raw.slice(offset); + const encrypted = raw.subarray(offset); timingsMs.preCrypt = performance.now(); - const key = await scryptAsync(this.secret, salt, v.keyBytes); - const decipher = crypto.createDecipheriv(v.algorithm, key, iv, v.algOptions); - decipher.setAAD(aad); - decipher.setAuthTag(tag); - const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]); + let decrypted; + let err; + let success = false; + for await (const secret of this.secrets) { + const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes); + const decipher = crypto.createDecipheriv(v.algorithm, key, iv, v.algOptions); + decipher.setAAD(aad); + decipher.setAuthTag(tag); + + try { + decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]); + success = true; + break; + } catch (e) { + err = e; + continue; + } + } + if (!success) { + throw err; + } let payload; timingsMs.preCompress = timingsMs.postCrypt = performance.now(); @@ -268,12 +495,17 @@ class MysteryBox { break; } timingsMs.end = timingsMs.postCompress = performance.now(); + stats.serializedBytes = payload.byteLength; + if (compression) { + stats.compressedBytes = decrypted.byteLength; + } if (!payloadIsBuffer) { payload = JSON.parse(payload.toString('utf8')); } - this.logger.debug(_scope, 'statistics', { version, flags: this._prettyFlags(flags), ...MysteryBox._timingsLog(timingsMs) }); + stats.flags = this._prettyFlags(flags); + this.emit('statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs), ...packageInfo }); return payload; } @@ -281,8 +513,8 @@ class MysteryBox { /** * Pretty-print flag values - * @param {Number} flags - * @returns {String} + * @param {number} flags flags + * @returns {string} pretty flags */ _prettyFlags(flags) { const flagNames = Object.entries(this.Flags).reduce((acc, cur) => { @@ -298,8 +530,14 @@ class MysteryBox { /** * Everyone loves numbers. - * @param {Object} timingsMs - * @returns + * @param {object} timingsMs raw timings + * @param {number} timingsMs.start start + * @param {number} timingsMs.preCompress checkpoint + * @param {number} timingsMs.postCompress checkpoint + * @param {number} timingsMs.preCrypt checkpoint + * @param {number} timingsMs.postCrypt checkpoint + * @param {number} timingsMs.end checkpoint + * @returns {object} computed timings */ static _timingsLog({ start, preCompress, postCompress, preCrypt, postCrypt, end }) { return { @@ -311,7 +549,4 @@ class MysteryBox { } -// Expose for stubbing in tests -MysteryBox._test = { crypto }; - -module.exports = MysteryBox; \ No newline at end of file +module.exports = MysteryBox;