X-Git-Url: http://git.squeep.com/?a=blobdiff_plain;f=lib%2Fmystery-box.js;h=abef55c474a619a273e68e932f793058d05ff853;hb=acfec23d615f8a14a283efef17b050228b9f02f5;hp=2343c5e164f79f9d04629a97c366a1fbd0f58595;hpb=b9782b5dfca21a6da610eebf8bd7207f10256fbf;p=squeep-mystery-box diff --git a/lib/mystery-box.js b/lib/mystery-box.js index 2343c5e..abef55c 100644 --- a/lib/mystery-box.js +++ b/lib/mystery-box.js @@ -3,7 +3,6 @@ const crypto = require('crypto'); const zlib = require('zlib'); const { promisify } = require('util'); -const { base64ToBase64URL, base64URLToBase64 } = require('@squeep/base64url'); const common = require('./common'); const allVersions = require('./version-parameters'); const { performance } = require('perf_hooks'); @@ -63,23 +62,27 @@ class MysteryBox { /** * @param {Console} logger * @param {Object} options - * @param {String} options.encryptionSecret + * @param {String|String[]} options.encryptionSecret - if an array, will always encrypt with first secret, will attempt to decrypt with all; useful for rolling secrets * @param {Number=} options.defaultFlags */ constructor(logger, options = {}) { this.logger = logger; this.options = options; - this.secret = options.encryptionSecret; - if (!this.secret) { + this.secrets = common.ensureArray(options.encryptionSecret); + if (!this.secrets.length) { throw new Error('missing encryption secret'); } - // TODO: support secret rolling // Filter any unavailable algorithms const availableCiphers = crypto.getCiphers(); + const availableHashes = crypto.getHashes(); + // Add legacy scrypt to available hashes for filtering key derivations + availableHashes.push(allVersions.KD.SCRYPT); this.versionParameters = Object.entries(allVersions).reduce((acc, [v, p]) => { - if (availableCiphers.includes(p.algorithm)) { + const validCipher = availableCiphers.includes(p.algorithm); + const validKeyDeriver = availableHashes.includes(p.keyDeriver); + if (validCipher && validKeyDeriver) { acc[v] = p; // eslint-disable-line security/detect-object-injection } return acc; @@ -110,9 +113,155 @@ class MysteryBox { } + /** + * Generate key data. + */ + static async _keyFromSecret(deriver, secret, salt, keyBytes) { + switch (deriver) { + case allVersions.KD.SHAKE256: { + const hash = crypto.createHash(allVersions.KD.SHAKE256, { outputLength: keyBytes }); + hash.update(salt); + hash.update(secret); + return hash.digest(); + } + + case allVersions.KD.BLAKE2B512: { + const hash = crypto.createHash(allVersions.KD.BLAKE2B512); + hash.update(salt); + hash.update(secret); + const digest = hash.digest(); + // should assert that keyBytes <= 64 + // but not concerned about that for now + // until we have a new algorithm with bigger key size + return digest.subarray(0, keyBytes); + } + + case allVersions.KD.SCRYPT: + return scryptAsync(secret, salt, keyBytes); + + default: + throw new RangeError('unsupported key deriver'); + } + } + + + /** + * Return bits and bit mask for given number of encoded bytes. + * @param {Number} numBytes + * @returns {Object} + */ + static _versionHeaderBits(numBytes) { + // Round up to 8 bits in result, just to be proper. + const resultBits = (((numBytes + 7) >> 3) << 3) >>> 0; + return { + headerValue: ((0xff << (resultBits - numBytes + 1)) & 0xff) >>> 0, + headerMask: ((0xff << (resultBits - numBytes)) & 0xff) >>> 0, + }; + } + + + /** + * Parse a byte into the total number of bytes in the packed number, + * returning that and the new value for the first byte, to update in the + * buffer before parsing as an unsigned integer. + * Number of packed bytes is indicated by location of first leading 0 bit. + * Support for numbers larger than 127 is of dubious practicality, but here + * it is anyhow. + * @param {Number} firstByte + * @returns {Object} + */ + static _versionHeaderDecode(firstByte) { + for (let numBytes = 1; numBytes <= 8; numBytes++) { + const { + headerValue, + headerMask, + } = MysteryBox._versionHeaderBits(numBytes); + if (((firstByte & headerMask) >>> 0) === headerValue) { + const restMask = (~headerMask & 0xff) >>> 0; + return { + numBytes, + firstByte: (firstByte & restMask) >>> 0, + }; + } + } + // Nine bytes would be an extravagence. + throw new RangeError(`unsupported version header (0x${firstByte.toString(16)})`); + } + + + /** + * Decode leading bytes of buffer as version identifier. + * In the first byte, the position of the first unset bit indicates how + * many total bytes comprise the version, in big-endian encoding. + * Returns decoded version and number of bytes used to decode. + * Only supports up to 6-byte numbers, and of those, only up to 4398046511103. + * @param {Buffer} buf - N.B. will be mogrified + * @returns {Object} + */ + static _versionDecode(buf) { + const headerByte = buf.readUInt8(0); + const { numBytes: versionBytes, firstByte } = MysteryBox._versionHeaderDecode(headerByte); + if (versionBytes === 1) { + return { + version: firstByte, + versionBytes, + }; + } + + if (versionBytes > 6) { + throw new RangeError(`unsupported version (${versionBytes} bytes)`); + } + + // Otherwise, update the masked first byte and parse the rest of the buffer. + buf[0] = firstByte; + return { + version: buf.readUIntBE(0, versionBytes), + versionBytes, + }; + } + + + /** + * Encode a version identifier into a buffer of a variable number of bytes. + * @param {Number} version + * @returns {Object} + */ + static _versionEncode(version) { + let versionBytes = 0; + + if (version <= 0x7f) { // 0-127 + versionBytes = 1; + } else if (version <= 0x3fff) { // 128-16383 + versionBytes = 2; + } else if (version <= 0x1fffff) { // 16384-2097151 + versionBytes = 3; + } else if (version <= 0x0fffffff) { // 2097152-268435455 + versionBytes = 4; + } else if (version <= 0x07ffffffff) { // 268435456-34359738367 + versionBytes = 5; + } else if (version <= 0x03ffffffffff) { // 34359738368-4398046511103 + versionBytes = 6; + } else { + throw new RangeError('version too large to encode'); + } + + const buffer = Buffer.alloc(versionBytes); + buffer.writeUIntBE(version, 0, versionBytes); + const headerByte = ((0xff << (8 - versionBytes + 1)) & 0xff) >>> 0; + buffer[0] = (buffer[0] | headerByte) >>> 0; + + return { + buffer, + versionBytes, + }; + } + + /** * Put contents into a mysterious box. * @param {Object|Buffer} contents + * @param {Number=} version + * @param {Number=} flags * @returns {String} */ async pack(contents, version = this.bestVersion, flags = this.defaultFlags) { @@ -121,10 +270,16 @@ class MysteryBox { start: performance.now(), preCompress: 0, postCompress: 0, - preEncrypt: 0, - postEncrypt: 0, + preCrypt: 0, + postCrypt: 0, end: 0, }; + const stats = { + version, + flags: undefined, + serializedBytes: undefined, // original contents size in bytes + compressedBytes: undefined, // compressed contents size in bytes + }; if (!(version in this.versionParameters)) { throw new RangeError(`MysteryBox format version ${version} not supported`); @@ -132,16 +287,26 @@ class MysteryBox { // eslint-disable-next-line security/detect-object-injection const v = this.versionParameters[version]; - const { compression } = MysteryBox._decodeFlags(flags); + const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags); if (Buffer.isBuffer(contents)) { + // Ensure payloadIsBuffer flag is set when contents are indeed a Buffer flags |= this.Flags.BufferPayload; } else { - contents = JSON.stringify(contents); + if (payloadIsBuffer) { + // Flag is set, but contents are not a Buffer? Try to coerce contents. + contents = Buffer.from(contents); + } else { + // Otherwise attempt to serialize the object + contents = JSON.stringify(contents); + } } + stats.serializedBytes = Buffer.byteLength(contents); - const versionBuffer = Buffer.alloc(v.versionBytes); - versionBuffer.writeUInt8(v.version, 0); + const { buffer: versionBuffer, versionBytes } = MysteryBox._versionEncode(v.version); + if (versionBytes !== v.versionBytes) { + throw new Error('internal inconsistency, mismatched version byte length'); + } const [iv, salt] = await Promise.all([ v.ivBytes, @@ -161,33 +326,40 @@ class MysteryBox { compressedContents = await deflateRawAsync(contents); break; } + stats.compressedBytes = Buffer.byteLength(compressedContents); + // const compressionRatio = stats.compressedBytes / stats.serializedBytes; timingsMs.postCompress = timingsMs.preCrypt = performance.now(); let payload; - if (compressedContents.length >= contents.length) { + if (stats.compressedBytes >= stats.serializedBytes) { + // If compression is not beneficial enough, or detrimental, do not use flags = flags & ~compressionFlagsMask; payload = contents; + stats.compressedBytes = undefined; } else { payload = compressedContents; } + const flagsBuffer = Buffer.alloc(v.flagsBytes); flagsBuffer.writeUInt8(flags, 0); + stats.flags = this._prettyFlags(flags); // Authenticate all this data const aadBuffer = Buffer.concat([versionBuffer, flagsBuffer, iv, salt]); - const key = await scryptAsync(this.secret, salt, v.keyBytes); + // Always encrypt with first secret + const secret = this.secrets[0]; + const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes); const cipher = crypto.createCipheriv(v.algorithm, key, iv, v.algOptions); cipher.setAAD(aadBuffer); const encrypted = cipher.update(payload); const final = cipher.final(); const tag = cipher.getAuthTag(); - const merged = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64'); - const result = base64ToBase64URL(merged); + const result = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64url'); timingsMs.end = timingsMs.postCrypt = performance.now(); - this.logger.debug(_scope, 'statistics', { version, flags: this._prettyFlags(flags), serialized: contents.length, compressed: payload.length, encoded: result.length, ...MysteryBox._timingsLog(timingsMs) }); + this.logger.debug(_scope, 'statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs) }); return result; } @@ -213,15 +385,19 @@ class MysteryBox { throw new RangeError('nothing to unpack'); } - const raw = Buffer.from(base64URLToBase64(box), 'base64'); + const raw = Buffer.from(box, 'base64url'); let offset = 0; - const version = raw.slice(offset, 1).readUInt8(0); + const { version, versionBytes } = MysteryBox._versionDecode(raw); if (!(version in this.versionParameters)) { throw new RangeError('unsupported version'); } // eslint-disable-next-line security/detect-object-injection const v = this.versionParameters[version]; + + if (v.versionBytes !== versionBytes) { + throw new Error('internal inconsistency, mismatched version byte length'); + } offset += v.versionBytes; const minBytes = v.versionBytes + v.flagsBytes + v.ivBytes + v.saltBytes + v.tagBytes; @@ -229,31 +405,47 @@ class MysteryBox { throw new RangeError('not enough to unpack'); } - const flags = raw.slice(offset, offset + v.flagsBytes).readUInt8(0); + const flags = raw.subarray(offset, offset + v.flagsBytes).readUInt8(0); offset += v.flagsBytes; const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags); - const iv = raw.slice(offset, offset + v.ivBytes); + const iv = raw.subarray(offset, offset + v.ivBytes); offset += v.ivBytes; - const salt = raw.slice(offset, offset + v.saltBytes); + const salt = raw.subarray(offset, offset + v.saltBytes); offset += v.saltBytes; - const aad = raw.slice(0, offset); // Everything up to here + const aad = raw.subarray(0, offset); // Everything up to here - const tag = raw.slice(offset, offset + v.tagBytes); + const tag = raw.subarray(offset, offset + v.tagBytes); offset += v.tagBytes; - const encrypted = raw.slice(offset); + const encrypted = raw.subarray(offset); timingsMs.preCrypt = performance.now(); - const key = await scryptAsync(this.secret, salt, v.keyBytes); - const decipher = crypto.createDecipheriv(v.algorithm, key, iv, v.algOptions); - decipher.setAAD(aad); - decipher.setAuthTag(tag); - const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]); + let decrypted; + let err; + let success = false; + for await (const secret of this.secrets) { + const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes); + const decipher = crypto.createDecipheriv(v.algorithm, key, iv, v.algOptions); + decipher.setAAD(aad); + decipher.setAuthTag(tag); + + try { + decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]); + success = true; + break; + } catch (e) { + err = e; + continue; + } + } + if (!success) { + throw err; + } let payload; timingsMs.preCompress = timingsMs.postCrypt = performance.now();