From: Justin Wind Date: Tue, 14 Mar 2023 01:44:31 +0000 (-0700) Subject: support multi-byte version identifiers for boxes. minor internal cleanup/refactors. X-Git-Tag: v1.2.2~2 X-Git-Url: https://git.squeep.com/?a=commitdiff_plain;h=a699109203e01e888c975ad128b72e49ef9f6498;p=squeep-mystery-box support multi-byte version identifiers for boxes. minor internal cleanup/refactors. --- diff --git a/lib/mystery-box.js b/lib/mystery-box.js index 55967ee..abef55c 100644 --- a/lib/mystery-box.js +++ b/lib/mystery-box.js @@ -78,7 +78,7 @@ class MysteryBox { const availableCiphers = crypto.getCiphers(); const availableHashes = crypto.getHashes(); // Add legacy scrypt to available hashes for filtering key derivations - availableHashes.push('scrypt'); + availableHashes.push(allVersions.KD.SCRYPT); this.versionParameters = Object.entries(allVersions).reduce((acc, [v, p]) => { const validCipher = availableCiphers.includes(p.algorithm); const validKeyDeriver = availableHashes.includes(p.keyDeriver); @@ -118,15 +118,15 @@ class MysteryBox { */ static async _keyFromSecret(deriver, secret, salt, keyBytes) { switch (deriver) { - case 'shake256': { - const hash = crypto.createHash('shake256', { outputLength: keyBytes }); + case allVersions.KD.SHAKE256: { + const hash = crypto.createHash(allVersions.KD.SHAKE256, { outputLength: keyBytes }); hash.update(salt); hash.update(secret); return hash.digest(); } - case 'blake2b512': { - const hash = crypto.createHash('blake2b512'); + case allVersions.KD.BLAKE2B512: { + const hash = crypto.createHash(allVersions.KD.BLAKE2B512); hash.update(salt); hash.update(secret); const digest = hash.digest(); @@ -136,7 +136,7 @@ class MysteryBox { return digest.subarray(0, keyBytes); } - case 'scrypt': + case allVersions.KD.SCRYPT: return scryptAsync(secret, salt, keyBytes); default: @@ -145,9 +145,123 @@ class MysteryBox { } + /** + * Return bits and bit mask for given number of encoded bytes. + * @param {Number} numBytes + * @returns {Object} + */ + static _versionHeaderBits(numBytes) { + // Round up to 8 bits in result, just to be proper. + const resultBits = (((numBytes + 7) >> 3) << 3) >>> 0; + return { + headerValue: ((0xff << (resultBits - numBytes + 1)) & 0xff) >>> 0, + headerMask: ((0xff << (resultBits - numBytes)) & 0xff) >>> 0, + }; + } + + + /** + * Parse a byte into the total number of bytes in the packed number, + * returning that and the new value for the first byte, to update in the + * buffer before parsing as an unsigned integer. + * Number of packed bytes is indicated by location of first leading 0 bit. + * Support for numbers larger than 127 is of dubious practicality, but here + * it is anyhow. + * @param {Number} firstByte + * @returns {Object} + */ + static _versionHeaderDecode(firstByte) { + for (let numBytes = 1; numBytes <= 8; numBytes++) { + const { + headerValue, + headerMask, + } = MysteryBox._versionHeaderBits(numBytes); + if (((firstByte & headerMask) >>> 0) === headerValue) { + const restMask = (~headerMask & 0xff) >>> 0; + return { + numBytes, + firstByte: (firstByte & restMask) >>> 0, + }; + } + } + // Nine bytes would be an extravagence. + throw new RangeError(`unsupported version header (0x${firstByte.toString(16)})`); + } + + + /** + * Decode leading bytes of buffer as version identifier. + * In the first byte, the position of the first unset bit indicates how + * many total bytes comprise the version, in big-endian encoding. + * Returns decoded version and number of bytes used to decode. + * Only supports up to 6-byte numbers, and of those, only up to 4398046511103. + * @param {Buffer} buf - N.B. will be mogrified + * @returns {Object} + */ + static _versionDecode(buf) { + const headerByte = buf.readUInt8(0); + const { numBytes: versionBytes, firstByte } = MysteryBox._versionHeaderDecode(headerByte); + if (versionBytes === 1) { + return { + version: firstByte, + versionBytes, + }; + } + + if (versionBytes > 6) { + throw new RangeError(`unsupported version (${versionBytes} bytes)`); + } + + // Otherwise, update the masked first byte and parse the rest of the buffer. + buf[0] = firstByte; + return { + version: buf.readUIntBE(0, versionBytes), + versionBytes, + }; + } + + + /** + * Encode a version identifier into a buffer of a variable number of bytes. + * @param {Number} version + * @returns {Object} + */ + static _versionEncode(version) { + let versionBytes = 0; + + if (version <= 0x7f) { // 0-127 + versionBytes = 1; + } else if (version <= 0x3fff) { // 128-16383 + versionBytes = 2; + } else if (version <= 0x1fffff) { // 16384-2097151 + versionBytes = 3; + } else if (version <= 0x0fffffff) { // 2097152-268435455 + versionBytes = 4; + } else if (version <= 0x07ffffffff) { // 268435456-34359738367 + versionBytes = 5; + } else if (version <= 0x03ffffffffff) { // 34359738368-4398046511103 + versionBytes = 6; + } else { + throw new RangeError('version too large to encode'); + } + + const buffer = Buffer.alloc(versionBytes); + buffer.writeUIntBE(version, 0, versionBytes); + const headerByte = ((0xff << (8 - versionBytes + 1)) & 0xff) >>> 0; + buffer[0] = (buffer[0] | headerByte) >>> 0; + + return { + buffer, + versionBytes, + }; + } + + /** * Put contents into a mysterious box. * @param {Object|Buffer} contents + * @param {Number=} version + * @param {Number=} flags * @returns {String} */ async pack(contents, version = this.bestVersion, flags = this.defaultFlags) { @@ -156,10 +270,16 @@ class MysteryBox { start: performance.now(), preCompress: 0, postCompress: 0, - preEncrypt: 0, - postEncrypt: 0, + preCrypt: 0, + postCrypt: 0, end: 0, }; + const stats = { + version, + flags: undefined, + serializedBytes: undefined, // original contents size in bytes + compressedBytes: undefined, // compressed contents size in bytes + }; if (!(version in this.versionParameters)) { throw new RangeError(`MysteryBox format version ${version} not supported`); @@ -167,16 +287,26 @@ class MysteryBox { // eslint-disable-next-line security/detect-object-injection const v = this.versionParameters[version]; - const { compression } = MysteryBox._decodeFlags(flags); + const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags); if (Buffer.isBuffer(contents)) { + // Ensure payloadIsBuffer flag is set when contents are indeed a Buffer flags |= this.Flags.BufferPayload; } else { - contents = JSON.stringify(contents); + if (payloadIsBuffer) { + // Flag is set, but contents are not a Buffer? Try to coerce contents. + contents = Buffer.from(contents); + } else { + // Otherwise attempt to serialize the object + contents = JSON.stringify(contents); + } } + stats.serializedBytes = Buffer.byteLength(contents); - const versionBuffer = Buffer.alloc(v.versionBytes); - versionBuffer.writeUInt8(v.version, 0); + const { buffer: versionBuffer, versionBytes } = MysteryBox._versionEncode(v.version); + if (versionBytes !== v.versionBytes) { + throw new Error('internal inconsistency, mismatched version byte length'); + } const [iv, salt] = await Promise.all([ v.ivBytes, @@ -196,17 +326,23 @@ class MysteryBox { compressedContents = await deflateRawAsync(contents); break; } + stats.compressedBytes = Buffer.byteLength(compressedContents); + // const compressionRatio = stats.compressedBytes / stats.serializedBytes; timingsMs.postCompress = timingsMs.preCrypt = performance.now(); let payload; - if (compressedContents.length >= contents.length) { + if (stats.compressedBytes >= stats.serializedBytes) { + // If compression is not beneficial enough, or detrimental, do not use flags = flags & ~compressionFlagsMask; payload = contents; + stats.compressedBytes = undefined; } else { payload = compressedContents; } + const flagsBuffer = Buffer.alloc(v.flagsBytes); flagsBuffer.writeUInt8(flags, 0); + stats.flags = this._prettyFlags(flags); // Authenticate all this data const aadBuffer = Buffer.concat([versionBuffer, flagsBuffer, iv, salt]); @@ -223,7 +359,7 @@ class MysteryBox { const result = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64url'); timingsMs.end = timingsMs.postCrypt = performance.now(); - this.logger.debug(_scope, 'statistics', { version, flags: this._prettyFlags(flags), serialized: contents.length, compressed: payload.length, encoded: result.length, ...MysteryBox._timingsLog(timingsMs) }); + this.logger.debug(_scope, 'statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs) }); return result; } @@ -252,12 +388,16 @@ class MysteryBox { const raw = Buffer.from(box, 'base64url'); let offset = 0; - const version = raw.subarray(offset, 1).readUInt8(0); + const { version, versionBytes } = MysteryBox._versionDecode(raw); if (!(version in this.versionParameters)) { throw new RangeError('unsupported version'); } // eslint-disable-next-line security/detect-object-injection const v = this.versionParameters[version]; + + if (v.versionBytes !== versionBytes) { + throw new Error('internal inconsistency, mismatched version byte length'); + } offset += v.versionBytes; const minBytes = v.versionBytes + v.flagsBytes + v.ivBytes + v.saltBytes + v.tagBytes; diff --git a/lib/version-parameters.js b/lib/version-parameters.js index 5f449a7..afa8441 100644 --- a/lib/version-parameters.js +++ b/lib/version-parameters.js @@ -1,12 +1,16 @@ 'use strict'; const ALG = { + __proto__: null, + AES_256_GCM: 'aes-256-gcm', CHACHA20_POLY1305: 'chacha20-poly1305', XCHACHA20_POLY1305: 'xchacha20-poly1305', }; const KD = { + __proto__: null, + SCRYPT: 'scrypt', SHAKE256: 'shake256', BLAKE2B512: 'blake2b512', @@ -18,6 +22,8 @@ const KD = { * More preferable versions are numbered higher. */ const allVersions = { + __proto__: null, + 1: { version: 1, algorithm: ALG.AES_256_GCM, @@ -140,4 +146,15 @@ const allVersions = { }, }; +Object.defineProperties(allVersions, { + ALG: { + enumerable: false, + get: () => ALG, + }, + KD: { + enumerable: false, + get: () => KD, + }, +}); + module.exports = allVersions; \ No newline at end of file diff --git a/test/lib/mystery-box.js b/test/lib/mystery-box.js index c71f6ff..9f48ff6 100644 --- a/test/lib/mystery-box.js +++ b/test/lib/mystery-box.js @@ -13,6 +13,7 @@ describe('MysteryBox', function () { options = { encryptionSecret: 'this is not a very good secret', }; + mb = new MysteryBox(stubLogger, options); }); afterEach(function () { sinon.restore(); @@ -61,6 +62,178 @@ describe('MysteryBox', function () { }); }); // _keyFromSecret + describe('_versionHeaderDecode', function () { + function _check(firstByte, numBytes, value) { + const result = MysteryBox._versionHeaderDecode(firstByte); + assert.strictEqual(result.numBytes, numBytes); + assert.strictEqual(result.firstByte, value); + } + it('decodes single byte, min', function () { + _check(0x00, 1, 0x00); + }); + it('decodes single byte, max', function () { + _check(0x7f, 1, 0x7f); + }); + it('decodes double byte, min', function () { + _check(0x80, 2, 0x00); + }); + it('decodes double byte, max', function () { + _check(0xbf, 2, 0x3f); + }); + it('decodes triple byte, min', function () { + _check(0xc0, 3, 0x00); + }); + it('decodes triple byte, max', function () { + _check(0xdf, 3, 0x1f); + }); + it('decodes quadruple byte, min', function () { + _check(0xe0, 4, 0x00); + }); + it('decodes quadruple byte, max', function () { + _check(0xef, 4, 0x0f); + }); + it('decodes quintuple byte, min', function () { + _check(0xf0, 5, 0x00); + }); + it('decodes quintuple byte, max', function () { + _check(0xf7, 5, 0x07); + }); + it('decodes sextuple byte, min', function () { + _check(0xf8, 6, 0x00); + }); + it('decodes sextuple byte, max', function () { + _check(0xfb, 6, 0x03); + }); + it('decodes septuple byte, min', function () { + _check(0xfc, 7, 0x00); + }); + it('decodes septuple byte, max', function () { + _check(0xfd, 7, 0x01); + }); + it('decodes double byte, min', function () { + _check(0xfe, 8, 0x00); + }); + it('decodes double byte, max', function () { + _check(0xfe, 8, 0x00); + }); + it('covers unsupported', function () { + assert.throws(() => MysteryBox._versionHeaderDecode(0xff), RangeError); + }); + }); // _versionHeaderDecode + + describe('_versionDecode', function () { + function _checkDecodeRange(start, end, numBytes, headerByte) { + const headerMask = ((0xff << (8 - numBytes)) & 0xff) >>> 0; + const hByte = ((0xff << (8 - numBytes + 1)) & 0xff) >>> 0; + assert.strictEqual(hByte, headerByte, `TEST ERROR: unexpected header for length, computed: ${hByte.toString(16)} passed: ${headerByte.toString(16)}`); + for (let v = start; v <= end; v++) { + const buffer = Buffer.alloc(numBytes); + buffer.writeUIntBE(v, 0, numBytes); + assert((buffer[0] & headerMask) === 0, `TEST ERROR: version ${v} encroached on headerByte 0x${headerByte.toString(16)} (${headerByte.toString(2)} & ${buffer[0].toString(2)})`); + buffer[0] = (buffer[0] | headerByte) >>> 0; + const { version, versionBytes } = MysteryBox._versionDecode(buffer); + assert.strictEqual(versionBytes, numBytes); + assert.strictEqual(version, v); + } + } + it('covers single-byte versions', function () { + _checkDecodeRange(0, 127, 1, 0x00); + }); + it('covers double-byte versions', function () { + _checkDecodeRange(128, 136, 2, 0x80); + // ... + _checkDecodeRange(16375, 16383, 2, 0x80); + }); + it('covers triple-byte versions', function () { + _checkDecodeRange(16384, 16390, 3, 0xc0); + // ... + _checkDecodeRange(2097145, 2097151, 3, 0xc0); + }); + it('covers quadruple-byte versions', function () { + _checkDecodeRange(2097151, 2097160, 4, 0xe0); + // ... + _checkDecodeRange(268435445, 268435455, 4, 0xe0); + }); + it('covers quintuple-byte versions', function () { + _checkDecodeRange(268435445, 268435445, 5, 0xf0); + // ... + _checkDecodeRange(34359738360, 34359738367, 5, 0xf0); + }); + it('covers sextuple-byte versions', function () { + _checkDecodeRange(34359738367, 34359738375, 6, 0xf8); + // ... + _checkDecodeRange(4398046511093, 4398046511103, 6, 0xf8); + }); + it('covers too big', function () { + const buffer = Buffer.from([0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + assert.throws(() => MysteryBox._versionDecode(buffer), RangeError); + }); + }); // _versionDecode + + describe('_versionEncode', function () { + function _checkEncodeRange(start, end, bytes, headerBits) { + for (let version = start; version <= end; version++) { + const expected = Buffer.alloc(bytes); + expected.writeUIntBE(version, 0, bytes); + expected[0] = (expected[0] | headerBits) >>> 0; + + const { buffer, versionBytes } = MysteryBox._versionEncode(version); + assert.deepStrictEqual(versionBytes, bytes, `version ${version} has ${versionBytes} bytes instead of expected ${bytes}`); + assert.deepStrictEqual(buffer, expected, `version ${version} buffer not expected: ${JSON.stringify(buffer)} vs ${JSON.stringify(expected)}`); + } + } + function _cheeckReciprical(version) { + const { buffer, versionBytes: encVB } = MysteryBox._versionEncode(version); + const { version: decV, versionBytes: decVB } = MysteryBox._versionDecode(buffer); + assert.strictEqual(encVB, decVB, `differing lengths for ${version}: enc:${encVB} dec:${decVB}`); + assert.strictEqual(decV, version, `failed for ${version}: ${JSON.stringify({ buffer, versionBytes: encVB })}`); + } + it('covers single-byte-packable versions', function () { + _checkEncodeRange(0, 127, 1, 0x00); + }); + it('covers double-byte-packable versions', function () { + _checkEncodeRange(128, 200, 2, 0x80); + /* ... */ + _checkEncodeRange(16380, 16383, 2, 0x80); + }); + it('covers triple-byte-packable versions', function () { + _checkEncodeRange(16384, 16390, 3, 0xc0); + /* ... */ + _checkEncodeRange(2097141, 2097151, 3, 0xc0); + }); + it('covers quadruple-byte-packable versions', function () { + _checkEncodeRange(2097152, 2097161, 4, 0xe0); + /* ... */ + _checkEncodeRange(268435445, 268435455, 4, 0xe0); + }); + it('covers quintuple-byte-packable versions', function () { + _checkEncodeRange(268435456, 268435465, 5, 0xf0) + /* ... */ + _checkEncodeRange(4294967294, 4294967296, 5, 0xf0) + /* ... */ + _checkEncodeRange(34359738360, 34359738367, 5, 0xf0) + }); + it('covers sextuple-byte-packable versions', function () { + _checkEncodeRange(34359738368, 34359738377, 6, 0xf8) + /* ... */ + _checkEncodeRange(4398046511093, 4398046511103, 6, 0xf8) + }); + it('covers too large', function () { + const version = 277076930199552; + assert.rejects(() => MysteryBox._versionEncode(version), RangeError); + }); + it('recipricates', function () { + [ + 0, 127, + 128, 16383, + 16384, 2097151, + 2097152, 268435455, + 268435456, 34359738367, + 34359738368, 4398046511103, + ].forEach((v) => _cheeckReciprical(v)); + }); + }); // _versionEncode + describe('pack, unpack', function () { beforeEach(function () { mb = new MysteryBox(stubLogger, options); @@ -97,6 +270,14 @@ describe('MysteryBox', function () { assert.deepStrictEqual(decryptedResult, object); }); + it('encrypts and decrypts default version, coerced buffer contents', async function () { + this.slow(500); + object = 'a string a string'; + const encryptedResult = await mb.pack(object, undefined, mb.Flags.BufferPayload | mb.defaultFlags); + const decryptedResult = await mb.unpack(encryptedResult); + assert.deepStrictEqual(decryptedResult, Buffer.from(object)); + }); + it('decrypts secondary (older) secret', async function () { this.slow(500); const oldmb = new MysteryBox(stubLogger, { encryptionSecret: 'old secret' }); @@ -190,6 +371,19 @@ describe('MysteryBox', function () { assert.rejects(() => mb.unpack(encryptedResult.slice(0, 6)), RangeError); }); + it('covers internal error, incorrect version byte size, pack', async function () { + const version = 1; + sinon.stub(mb.versionParameters[version], 'versionBytes').value(10); + assert.rejects(() => mb.pack({}, version), Error); + }); + + it('covers internal error, incorrect version byte size, unpack', async function () { + const version = 1; + const encryptedResult = await mb.pack({}, version); + sinon.stub(mb.versionParameters[version], 'versionBytes').value(10); + assert.rejects(() => mb.unpack(encryptedResult), Error); + }); + }); // pack, unpack }); // MysteryBox \ No newline at end of file diff --git a/test/lib/version-parameters.js b/test/lib/version-parameters.js new file mode 100644 index 0000000..a245627 --- /dev/null +++ b/test/lib/version-parameters.js @@ -0,0 +1,16 @@ +/* eslint-env mocha */ +'use strict'; + +const assert = require('assert'); +const vp = require('../../lib/version-parameters'); + +describe('Version Parameters', function () { + it('covers ALG getter', function () { + const alg = vp.ALG; + assert.notEqual(Object.keys(alg).length, 0); + }); + it('covers KD getter', function () { + const kd = vp.KD; + assert.notEqual(Object.keys(kd).length, 0); + }); +}); // Version Parameters