git.squeep.com Git - squeep-mystery-box/blob - lib/mystery-box.js

   1 'use strict';
   2
   3 const { EventEmitter } = require('events');
   4 const crypto = require('crypto');
   5 const zlib = require('zlib');
   6 const { promisify } = require('util');
   7 const common = require('./common');
   8 const { MysteryBoxError } = require('./errors');
   9 const allVersions = require('./version-parameters');
  10 const { performance } = require('perf_hooks');
  11 const { name: packageName, version: packageVersion } = require('../package.json');
  12
  13 const packageInfo = {
  14   packageName,
  15   packageVersion,
  16 };
  17
  18 const brotliCompressAsync = promisify(zlib.brotliCompress);
  19 const brotliDecompressAsync = promisify(zlib.brotliDecompress);
  20 const deflateRawAsync = promisify(zlib.deflateRaw);
  21 const inflateRawAsync = promisify(zlib.inflateRaw);
  22 const scryptAsync = promisify(crypto.scrypt);
  23
  24 /**
  25  * Only you will know what's inside your...
  26  *    \  |            |               | __ )
  27  *   |\/ | |   |  __| __|  _ \  __|   | __ \   _ \ \  /
  28  *   |   | |   |\__ \ |    __/ |  |   | |   | (   |`  <
  29  *  _|  _|\__, |____/\__|\___|_| \__, |____/ \___/ _/\_\
  30  *        ____/                  ____/
  31  *
  32  * Our very own way of converting a buffer or serializable object
  33  * to and from an opaque and web-safe representation, which we can
  34  * let anyone see without disclosing the contents, nor allowing it
  35  * to be modified without detection.
  36  *
  37  * The result is a Base64URL-encoded byte-array consisting of:
  38  * - version, indicating format and encryption algorithm
  39  * - flags, indicating state of the contents
  40  * - iv/nonce, randomness for the algorithm
  41  * - salt, applied to a secret to derive a key
  42  * - tag, additional room for encryption to validate the previous fields
  43  * - payload, encrypted version of possibly compressed and serialized object
  44  *
  45  */
  46
  47
  48 const availableFlags = {
  49   Brotli: (1<<0), // Slightly better compression, but slow
  50   Flate: (1<<1), // Slightly worse compression, but much faster especially with larger payloads.  Prefer this.
  51   FutureCompression: (1<<0)|(1<<1), // Something else, like zstd maybe.
  52
  53   Unused2: (1<<2),
  54   Unused3: (1<<3),
  55   Unused4: (1<<4),
  56   Unused5: (1<<5),
  57   Unused6: (1<<6),
  58
  59   BufferPayload: (1<<7), // Payload is a buffer, not an object
  60 };
  61 const compressionFlagsMask = (availableFlags.Flate | availableFlags.Brotli);
  62 const compressionFlagsShift = 0;
  63 const payloadFlagsMask = (availableFlags.BufferPayload);
  64 const payloadFlagsShift = 7;
  65
  66
  67 class MysteryBox extends EventEmitter {
  68   /**
  69    * @param {Object} options
  70    * @param {String|String[]} options.encryptionSecret - if an array, will always encrypt with first secret, will attempt to decrypt with all; useful for rolling secrets
  71    * @param {Number=} options.defaultFlags
  72    */
  73   constructor(options = {}, ...args) {
  74     super(...args);
  75     this.secrets = common.ensureArray(options.encryptionSecret);
  76     if (!this.secrets.length) {
  77       throw new MysteryBoxError('missing encryption secret');
  78     }
  79
  80     // Filter any unavailable algorithms
  81     const availableCiphers = crypto.getCiphers();
  82     const availableHashes = crypto.getHashes();
  83     // Add legacy scrypt to available hashes for filtering key derivations
  84     availableHashes.push(allVersions.KD.SCRYPT);
  85     this.versionParameters = Object.entries(allVersions).reduce((acc, [v, p]) => {
  86       const validCipher = availableCiphers.includes(p.algorithm);
  87       const validKeyDeriver = availableHashes.includes(p.keyDeriver);
  88       if (validCipher && validKeyDeriver) {
  89         acc[v] = p; // eslint-disable-line security/detect-object-injection
  90       }
  91       return acc;
  92     }, {});
  93
  94     // Default to highest
  95     this.bestVersion = Number(Object.keys(this.versionParameters).sort().pop());
  96     if (Number.isNaN(this.bestVersion)) {
  97       throw new MysteryBoxError('no supported versions available');
  98     }
  99
 100     this.Flags = availableFlags;
 101     this.defaultFlags = 'defaultFlags' in options ? options.defaultFlags : availableFlags.Flate;
 102     if (this.defaultFlags < 0 || this.defaultFlags > 255) {
 103       throw new MysteryBoxError('Invalid default flag value');
 104     }
 105   }
 106
 107
 108   /**
 109    * Parse the bits out of the flags.
 110    */
 111   static _decodeFlags(flags) {
 112     return {
 113       compression: (flags & compressionFlagsMask) >> compressionFlagsShift,
 114       payloadIsBuffer: Boolean((flags & payloadFlagsMask) >> payloadFlagsShift),
 115     };
 116   }
 117
 118
 119   /**
 120    * Generate key data.
 121    */
 122   static async _keyFromSecret(deriver, secret, salt, keyBytes) {
 123     switch (deriver) {
 124       case allVersions.KD.SHAKE256: {
 125         const hash = crypto.createHash(allVersions.KD.SHAKE256, { outputLength: keyBytes });
 126         hash.update(salt);
 127         hash.update(secret);
 128         return hash.digest();
 129       }
 130
 131       case allVersions.KD.BLAKE2B512: {
 132         const hash = crypto.createHash(allVersions.KD.BLAKE2B512);
 133         hash.update(salt);
 134         hash.update(secret);
 135         const digest = hash.digest();
 136         // should assert that keyBytes <= 64
 137         // but not concerned about that for now
 138         // until we have a new algorithm with bigger key size
 139         return digest.subarray(0, keyBytes);
 140       }
 141
 142       case allVersions.KD.SCRYPT:
 143         return scryptAsync(secret, salt, keyBytes);
 144
 145       default:
 146         throw new MysteryBoxError('unsupported key deriver');
 147     }
 148   }
 149
 150
 151   /**
 152    * Return bits and bit mask for given number of encoded bytes.
 153    * @param {Number} numBytes
 154    * @returns {Object}
 155    */
 156   static _versionHeaderBits(numBytes) {
 157     // Round up to 8 bits in result, just to be proper.
 158     const resultBits = (((numBytes + 7) >> 3) << 3) >>> 0;
 159     return {
 160       headerValue: ((0xff << (resultBits - numBytes + 1)) & 0xff) >>> 0,
 161       headerMask: ((0xff << (resultBits - numBytes)) & 0xff) >>> 0,
 162     };
 163   }
 164
 165
 166   /**
 167    * Parse a byte into the total number of bytes in the packed number,
 168    * returning that and the new value for the first byte, to update in the
 169    * buffer before parsing as an unsigned integer.
 170    * Number of packed bytes is indicated by location of first leading 0 bit.
 171    * Support for numbers larger than 127 is of dubious practicality, but here
 172    * it is anyhow.
 173    * @param {Number} firstByte
 174    * @returns {Object}
 175    */
 176   static _versionHeaderDecode(firstByte) {
 177     for (let numBytes = 1; numBytes <= 8; numBytes++) {
 178       const {
 179         headerValue,
 180         headerMask,
 181       } = MysteryBox._versionHeaderBits(numBytes);
 182       if (((firstByte & headerMask) >>> 0) === headerValue) {
 183         const restMask = (~headerMask & 0xff) >>> 0;
 184         return {
 185           numBytes,
 186           firstByte: (firstByte & restMask) >>> 0,
 187         };
 188       }
 189     }
 190     // Nine bytes would be an extravagence.
 191     throw new MysteryBoxError(`unsupported version header (0x${firstByte.toString(16)})`);
 192   }
 193
 194
 195   /**
 196    * Decode leading bytes of buffer as version identifier.
 197    * In the first byte, the position of the first unset bit indicates how
 198    * many total bytes comprise the version, in big-endian encoding.
 199    * Returns decoded version and number of bytes used to decode.
 200    * Only supports up to 6-byte numbers, and of those, only up to 4398046511103.
 201    * @param {Buffer} buf - N.B. will be mogrified
 202    * @returns {Object}
 203    */
 204   static _versionDecode(buf) {
 205     const headerByte = buf.readUInt8(0);
 206     const { numBytes: versionBytes, firstByte } = MysteryBox._versionHeaderDecode(headerByte);
 207     if (versionBytes === 1) {
 208       return {
 209         version: firstByte,
 210         versionBytes,
 211       };
 212     }
 213
 214     if (versionBytes > 6) {
 215       throw new MysteryBoxError(`unsupported version (${versionBytes} bytes)`);
 216     }
 217
 218     // Otherwise, update the masked first byte and parse the rest of the buffer.
 219     buf[0] = firstByte;
 220     return {
 221       version: buf.readUIntBE(0, versionBytes),
 222       versionBytes,
 223     };
 224   }
 225
 226
 227   /**
 228    * Encode a version identifier into a buffer of a variable number of bytes.
 229    * @param {Number} version
 230    * @returns {Object}
 231    */
 232   static _versionEncode(version) {
 233     let versionBytes = 0;
 234
 235     if (version <= 0x7f) { // 0-127
 236       versionBytes = 1;
 237     } else if (version <= 0x3fff) { // 128-16383
 238       versionBytes = 2;
 239     } else if (version <= 0x1fffff) { // 16384-2097151
 240       versionBytes = 3;
 241     } else if (version <= 0x0fffffff) { // 2097152-268435455
 242       versionBytes = 4;
 243     } else if (version <= 0x07ffffffff) { // 268435456-34359738367
 244       versionBytes = 5;
 245     } else if (version <= 0x03ffffffffff) { // 34359738368-4398046511103
 246       versionBytes = 6;
 247     } else {
 248       throw new MysteryBoxError(`version too large to encode (${version})`);
 249     }
 250
 251     const buffer = Buffer.alloc(versionBytes);
 252     buffer.writeUIntBE(version, 0, versionBytes);
 253     const headerByte = ((0xff << (8 - versionBytes + 1)) & 0xff) >>> 0;
 254     buffer[0] = (buffer[0] | headerByte) >>> 0;
 255
 256     return {
 257       buffer,
 258       versionBytes,
 259     };
 260   }
 261
 262
 263   /**
 264    * Stats tracked when packing/unpacking boxes.
 265    * @returns {Object}
 266    */
 267   static _newStats(method) {
 268     return {
 269       stats: {
 270         method,
 271         version: undefined,
 272         flags: undefined,
 273         flagsRaw: undefined,
 274         serializedBytes: undefined,
 275         compressedBytes: undefined,
 276       },
 277       timingsMs: {
 278         start: performance.now(),
 279         preCompress: 0,
 280         postCompress: 0,
 281         preCrypt: 0,
 282         postCrypt: 0,
 283         end: 0,
 284       },
 285     };
 286   }
 287
 288
 289   /**
 290    * Put contents into a mysterious box.
 291    * @param {Object|Buffer} contents
 292    * @param {Number=} version
 293    * @param {Number=} flags
 294    * @returns {String}
 295    */
 296   async pack(contents, version = this.bestVersion, flags = this.defaultFlags) {
 297     const { stats, timingsMs } = MysteryBox._newStats('pack');
 298
 299     if (!(version in this.versionParameters)) {
 300       throw new MysteryBoxError(`MysteryBox format version ${version} not supported`);
 301     }
 302     // eslint-disable-next-line security/detect-object-injection
 303     const v = this.versionParameters[version];
 304     stats.version = version;
 305
 306     const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags);
 307
 308     if (Buffer.isBuffer(contents)) {
 309       // Ensure payloadIsBuffer flag is set when contents are indeed a Buffer
 310       flags |= this.Flags.BufferPayload;
 311     } else {
 312       if (payloadIsBuffer) {
 313         // Flag is set, but contents are not a Buffer? Try to coerce contents.
 314         contents = Buffer.from(contents);
 315       } else {
 316         // Otherwise attempt to serialize the object
 317         contents = JSON.stringify(contents);
 318       }
 319     }
 320     stats.serializedBytes = Buffer.byteLength(contents);
 321
 322     const { buffer: versionBuffer, versionBytes } = MysteryBox._versionEncode(v.version);
 323     if (versionBytes !== v.versionBytes) {
 324       throw new MysteryBoxError('internal inconsistency, mismatched version byte length');
 325     }
 326
 327     const [iv, salt] = await Promise.all([
 328       v.ivBytes,
 329       v.saltBytes,
 330     ].map((b) => common.randomBytesAsync(b)));
 331
 332     timingsMs.preCompress = performance.now();
 333     let compressedContents;
 334     switch (compression) {
 335       case 0: // No compression requested
 336         compressedContents = contents;
 337         break;
 338       case this.Flags.Brotli:
 339         compressedContents = await brotliCompressAsync(contents);
 340         break;
 341       case this.Flags.Flate:
 342         compressedContents = await deflateRawAsync(contents);
 343         break;
 344     }
 345     stats.compressedBytes = Buffer.byteLength(compressedContents);
 346     // const compressionRatio = stats.compressedBytes / stats.serializedBytes;
 347     timingsMs.postCompress = timingsMs.preCrypt = performance.now();
 348
 349     let payload;
 350     if (stats.compressedBytes >= stats.serializedBytes) {
 351       // If compression is not beneficial enough, or detrimental, do not use
 352       flags = flags & ~compressionFlagsMask;
 353       payload = contents;
 354       stats.compressedBytes = undefined;
 355     } else {
 356       payload = compressedContents;
 357     }
 358
 359     const flagsBuffer = Buffer.alloc(v.flagsBytes);
 360     flagsBuffer.writeUInt8(flags, 0);
 361     stats.flagsRaw = flags;
 362     stats.flags = this._prettyFlags(flags);
 363
 364     // Authenticate all this data
 365     const aadBuffer = Buffer.concat([versionBuffer, flagsBuffer, iv, salt]);
 366
 367     // Always encrypt with first secret
 368     const secret = this.secrets[0];
 369     const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes);
 370     const cipher = crypto.createCipheriv(v.algorithm, key, iv, v.algOptions);
 371     cipher.setAAD(aadBuffer);
 372     const encrypted = cipher.update(payload);
 373     const final = cipher.final();
 374     const tag = cipher.getAuthTag();
 375
 376     const result = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64url');
 377     timingsMs.end = timingsMs.postCrypt = performance.now();
 378
 379     this.emit('statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs), ...packageInfo });
 380
 381     return result;
 382   }
 383
 384
 385   /**
 386    * Take contents out of a mysterious box.
 387    * @param {String} box - Base64URL encoded payload
 388    * @returns {Object}
 389    */
 390   async unpack(box) {
 391     const { stats, timingsMs } = MysteryBox._newStats('unpack');
 392
 393     if (!box) {
 394       throw new MysteryBoxError('nothing to unpack');
 395     }
 396
 397     const raw = Buffer.from(box, 'base64url');
 398     let offset = 0;
 399
 400     const { version, versionBytes } = MysteryBox._versionDecode(raw);
 401     if (!(version in this.versionParameters)) {
 402       throw new MysteryBoxError(`unsupported version (${version})`);
 403     }
 404     // eslint-disable-next-line security/detect-object-injection
 405     const v = this.versionParameters[version];
 406
 407     if (v.versionBytes !== versionBytes) {
 408       throw new MysteryBoxError('internal inconsistency, mismatched version byte length');
 409     }
 410     offset += v.versionBytes;
 411     stats.version = version;
 412
 413     const minBytes = v.versionBytes + v.flagsBytes + v.ivBytes + v.saltBytes + v.tagBytes;
 414     if (raw.length < minBytes) {
 415       throw new MysteryBoxError('not enough to unpack');
 416     }
 417
 418     const flags = raw.subarray(offset, offset + v.flagsBytes).readUInt8(0);
 419     offset += v.flagsBytes;
 420     stats.flagsRaw = flags;
 421
 422     const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags);
 423
 424     const iv = raw.subarray(offset, offset + v.ivBytes);
 425     offset += v.ivBytes;
 426
 427     const salt = raw.subarray(offset, offset + v.saltBytes);
 428     offset += v.saltBytes;
 429
 430     const aad = raw.subarray(0, offset); // Everything up to here
 431
 432     const tag = raw.subarray(offset, offset + v.tagBytes);
 433     offset += v.tagBytes;
 434
 435     const encrypted = raw.subarray(offset);
 436
 437     timingsMs.preCrypt = performance.now();
 438
 439     let decrypted;
 440     let err;
 441     let success = false;
 442     for await (const secret of this.secrets) {
 443       const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes);
 444       const decipher = crypto.createDecipheriv(v.algorithm, key, iv, v.algOptions);
 445       decipher.setAAD(aad);
 446       decipher.setAuthTag(tag);
 447
 448       try {
 449         decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
 450         success = true;
 451         break;
 452       } catch (e) {
 453         err = e;
 454         continue;
 455       }
 456     }
 457     if (!success) {
 458       throw err;
 459     }
 460
 461     let payload;
 462     timingsMs.preCompress = timingsMs.postCrypt = performance.now();
 463     switch (compression) {
 464       case 0: // No compression
 465         payload = decrypted;
 466         break;
 467       case this.Flags.Brotli:
 468         payload = await brotliDecompressAsync(decrypted);
 469         break;
 470       case this.Flags.Flate:
 471         payload = await inflateRawAsync(decrypted);
 472         break;
 473     }
 474     timingsMs.end = timingsMs.postCompress = performance.now();
 475     stats.serializedBytes = payload.byteLength;
 476     if (compression) {
 477       stats.compressedBytes = decrypted.byteLength;
 478     }
 479
 480     if (!payloadIsBuffer) {
 481       payload = JSON.parse(payload.toString('utf8'));
 482     }
 483
 484     stats.flags = this._prettyFlags(flags);
 485     this.emit('statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs), ...packageInfo });
 486
 487     return payload;
 488   }
 489
 490
 491   /**
 492    * Pretty-print flag values
 493    * @param {Number} flags
 494    * @returns {String}
 495    */
 496   _prettyFlags(flags) {
 497     const flagNames = Object.entries(this.Flags).reduce((acc, cur) => {
 498       const [flagName, flagValue] = cur;
 499       if ((flags & flagValue) === flagValue) {
 500         acc.push(flagName);
 501       }
 502       return acc;
 503     }, []);
 504     return `0x${flags.toString(16).padStart(2, '0')} [${flagNames.join(',')}]`;
 505   }
 506
 507
 508   /**
 509    * Everyone loves numbers.
 510    * @param {Object} timingsMs
 511    * @returns
 512    */
 513   static _timingsLog({ start, preCompress, postCompress, preCrypt, postCrypt, end }) {
 514     return {
 515       totalMs: end - start,
 516       compressMs: postCompress - preCompress,
 517       cryptMs: postCrypt - preCrypt,
 518     };
 519   }
 520
 521 }
 522
 523 module.exports = MysteryBox;