git.squeep.com Git - squeep-mystery-box/blob - lib/mystery-box.js

   1 'use strict';
   2
   3 const { EventEmitter } = require('events');
   4 const crypto = require('crypto');
   5 const zlib = require('zlib');
   6 const { promisify } = require('util');
   7 const { MysteryBoxError } = require('./errors');
   8 const allVersions = require('./version-parameters');
   9 const { performance } = require('perf_hooks');
  10 const { name: packageName, version: packageVersion } = require('../package.json');
  11
  12 const packageInfo = {
  13   packageName,
  14   packageVersion,
  15 };
  16
  17 const brotliCompressAsync = promisify(zlib.brotliCompress);
  18 const brotliDecompressAsync = promisify(zlib.brotliDecompress);
  19 const deflateRawAsync = promisify(zlib.deflateRaw);
  20 const inflateRawAsync = promisify(zlib.inflateRaw);
  21 const scryptAsync = promisify(crypto.scrypt);
  22 const randomBytesAsync = promisify(crypto.randomBytes);
  23
  24 /**
  25  * Only you will know what's inside your...
  26  *    \  |            |               | __ )
  27  *   |\/ | |   |  __| __|  _ \  __|   | __ \   _ \ \  /
  28  *   |   | |   |\__ \ |    __/ |  |   | |   | (   |`  <
  29  *  _|  _|\__, |____/\__|\___|_| \__, |____/ \___/ _/\_\
  30  *        ____/                  ____/
  31  *
  32  * Our very own way of converting a buffer or serializable object
  33  * to and from an opaque and web-safe representation, which we can
  34  * let anyone see without disclosing the contents, nor allowing it
  35  * to be modified without detection.
  36  *
  37  * The result is a Base64URL-encoded byte-array consisting of:
  38  * - version, indicating format and encryption algorithm
  39  * - flags, indicating state of the contents
  40  * - iv/nonce, randomness for the algorithm
  41  * - salt, applied to a secret to derive a key
  42  * - tag, additional room for encryption to validate the previous fields
  43  * - payload, encrypted version of possibly compressed and serialized object
  44  *
  45  */
  46
  47
  48 const availableFlags = {
  49   Brotli: (1<<0), // Slightly better compression, but slow
  50   Flate: (1<<1), // Slightly worse compression, but much faster especially with larger payloads.  Prefer this.
  51   FutureCompression: (1<<0)|(1<<1), // Something else, like zstd maybe.
  52
  53   Unused2: (1<<2),
  54   Unused3: (1<<3),
  55   Unused4: (1<<4),
  56   Unused5: (1<<5),
  57   Unused6: (1<<6),
  58
  59   BufferPayload: (1<<7), // Payload is a buffer, not an object
  60 };
  61 const compressionFlagsMask = (availableFlags.Flate | availableFlags.Brotli);
  62 const compressionFlagsShift = 0;
  63 const payloadFlagsMask = (availableFlags.BufferPayload);
  64 const payloadFlagsShift = 7;
  65
  66 class MysteryBox extends EventEmitter {
  67   /**
  68    * @param {Object} options
  69    * @param {String|String[]} options.encryptionSecret - if an array, will always encrypt with first secret, will attempt to decrypt with all; useful for rolling secrets
  70    * @param {Number=} options.defaultFlags
  71    */
  72   constructor(options = {}, ...args) {
  73     super(...args);
  74     this.secrets = MysteryBox._ensureArray(options.encryptionSecret);
  75     if (!this.secrets.length) {
  76       throw new MysteryBoxError('missing encryption secret');
  77     }
  78
  79     // Filter any unavailable algorithms
  80     const availableCiphers = crypto.getCiphers();
  81     const availableHashes = crypto.getHashes();
  82     // Add legacy scrypt to available hashes for filtering key derivations
  83     availableHashes.push(allVersions.KD.SCRYPT);
  84     this.versionParameters = Object.entries(allVersions).reduce((acc, [v, p]) => {
  85       const validCipher = availableCiphers.includes(p.algorithm);
  86       const validKeyDeriver = availableHashes.includes(p.keyDeriver);
  87       if (validCipher && validKeyDeriver) {
  88         acc[v] = p; // eslint-disable-line security/detect-object-injection
  89       }
  90       return acc;
  91     }, {});
  92
  93     // Default to highest
  94     this.bestVersion = Number(Object.keys(this.versionParameters).sort().pop());
  95     if (Number.isNaN(this.bestVersion)) {
  96       throw new MysteryBoxError('no supported versions available');
  97     }
  98
  99     this.Flags = availableFlags;
 100     this.defaultFlags = 'defaultFlags' in options ? options.defaultFlags : availableFlags.Flate;
 101     if (this.defaultFlags < 0 || this.defaultFlags > 255) {
 102       throw new MysteryBoxError('Invalid default flag value');
 103     }
 104   }
 105
 106
 107   /**
 108    * Return an array containing x if x is something and not an array
 109    * @param {*} x
 110    */
 111   static _ensureArray(x) {
 112     if (x === undefined) {
 113       return [];
 114     }
 115     if (!Array.isArray(x)) {
 116       return Array(x);
 117     }
 118     return x;
 119   }
 120
 121
 122   /**
 123    * Parse the bits out of the flags.
 124    */
 125   static _decodeFlags(flags) {
 126     return {
 127       compression: (flags & compressionFlagsMask) >> compressionFlagsShift,
 128       payloadIsBuffer: Boolean((flags & payloadFlagsMask) >> payloadFlagsShift),
 129     };
 130   }
 131
 132
 133   /**
 134    * Generate key data.
 135    */
 136   static async _keyFromSecret(deriver, secret, salt, keyBytes) {
 137     switch (deriver) {
 138       case allVersions.KD.SHAKE256: {
 139         const hash = crypto.createHash(allVersions.KD.SHAKE256, { outputLength: keyBytes });
 140         hash.update(salt);
 141         hash.update(secret);
 142         return hash.digest();
 143       }
 144
 145       case allVersions.KD.BLAKE2B512: {
 146         const hash = crypto.createHash(allVersions.KD.BLAKE2B512);
 147         hash.update(salt);
 148         hash.update(secret);
 149         const digest = hash.digest();
 150         // should assert that keyBytes <= 64
 151         // but not concerned about that for now
 152         // until we have a new algorithm with bigger key size
 153         return digest.subarray(0, keyBytes);
 154       }
 155
 156       case allVersions.KD.SCRYPT:
 157         return scryptAsync(secret, salt, keyBytes);
 158
 159       default:
 160         throw new MysteryBoxError('unsupported key deriver');
 161     }
 162   }
 163
 164
 165   /**
 166    * Return bits and bit mask for given number of encoded bytes.
 167    * @param {Number} numBytes
 168    * @returns {Object}
 169    */
 170   static _versionHeaderBits(numBytes) {
 171     // Round up to 8 bits in result, just to be proper.
 172     const resultBits = (((numBytes + 7) >> 3) << 3) >>> 0;
 173     return {
 174       headerValue: ((0xff << (resultBits - numBytes + 1)) & 0xff) >>> 0,
 175       headerMask: ((0xff << (resultBits - numBytes)) & 0xff) >>> 0,
 176     };
 177   }
 178
 179
 180   /**
 181    * Parse a byte into the total number of bytes in the packed number,
 182    * returning that and the new value for the first byte, to update in the
 183    * buffer before parsing as an unsigned integer.
 184    * Number of packed bytes is indicated by location of first leading 0 bit.
 185    * Support for numbers larger than 127 is of dubious practicality, but here
 186    * it is anyhow.
 187    * @param {Number} firstByte
 188    * @returns {Object}
 189    */
 190   static _versionHeaderDecode(firstByte) {
 191     for (let numBytes = 1; numBytes <= 8; numBytes++) {
 192       const {
 193         headerValue,
 194         headerMask,
 195       } = MysteryBox._versionHeaderBits(numBytes);
 196       if (((firstByte & headerMask) >>> 0) === headerValue) {
 197         const restMask = (~headerMask & 0xff) >>> 0;
 198         return {
 199           numBytes,
 200           firstByte: (firstByte & restMask) >>> 0,
 201         };
 202       }
 203     }
 204     // Nine bytes would be an extravagence.
 205     throw new MysteryBoxError(`unsupported version header (0x${firstByte.toString(16)})`);
 206   }
 207
 208
 209   /**
 210    * Decode leading bytes of buffer as version identifier.
 211    * In the first byte, the position of the first unset bit indicates how
 212    * many total bytes comprise the version, in big-endian encoding.
 213    * Returns decoded version and number of bytes used to decode.
 214    * Only supports up to 6-byte numbers, and of those, only up to 4398046511103.
 215    * @param {Buffer} buf - N.B. will be mogrified
 216    * @returns {Object}
 217    */
 218   static _versionDecode(buf) {
 219     const headerByte = buf.readUInt8(0);
 220     const { numBytes: versionBytes, firstByte } = MysteryBox._versionHeaderDecode(headerByte);
 221     if (versionBytes === 1) {
 222       return {
 223         version: firstByte,
 224         versionBytes,
 225       };
 226     }
 227
 228     if (versionBytes > 6) {
 229       throw new MysteryBoxError(`unsupported version (${versionBytes} bytes)`);
 230     }
 231
 232     // Otherwise, update the masked first byte and parse the rest of the buffer.
 233     buf[0] = firstByte;
 234     return {
 235       version: buf.readUIntBE(0, versionBytes),
 236       versionBytes,
 237     };
 238   }
 239
 240
 241   /**
 242    * Encode a version identifier into a buffer of a variable number of bytes.
 243    * @param {Number} version
 244    * @returns {Object}
 245    */
 246   static _versionEncode(version) {
 247     let versionBytes = 0;
 248
 249     if (version <= 0x7f) { // 0-127
 250       versionBytes = 1;
 251     } else if (version <= 0x3fff) { // 128-16383
 252       versionBytes = 2;
 253     } else if (version <= 0x1fffff) { // 16384-2097151
 254       versionBytes = 3;
 255     } else if (version <= 0x0fffffff) { // 2097152-268435455
 256       versionBytes = 4;
 257     } else if (version <= 0x07ffffffff) { // 268435456-34359738367
 258       versionBytes = 5;
 259     } else if (version <= 0x03ffffffffff) { // 34359738368-4398046511103
 260       versionBytes = 6;
 261     } else {
 262       throw new MysteryBoxError(`version too large to encode (${version})`);
 263     }
 264
 265     const buffer = Buffer.alloc(versionBytes);
 266     buffer.writeUIntBE(version, 0, versionBytes);
 267     const headerByte = ((0xff << (8 - versionBytes + 1)) & 0xff) >>> 0;
 268     buffer[0] = (buffer[0] | headerByte) >>> 0;
 269
 270     return {
 271       buffer,
 272       versionBytes,
 273     };
 274   }
 275
 276
 277   /**
 278    * Stats tracked when packing/unpacking boxes.
 279    * @returns {Object}
 280    */
 281   static _newStats(method) {
 282     return {
 283       stats: {
 284         method,
 285         version: undefined,
 286         flags: undefined,
 287         flagsRaw: undefined,
 288         serializedBytes: undefined,
 289         compressedBytes: undefined,
 290       },
 291       timingsMs: {
 292         start: performance.now(),
 293         preCompress: 0,
 294         postCompress: 0,
 295         preCrypt: 0,
 296         postCrypt: 0,
 297         end: 0,
 298       },
 299     };
 300   }
 301
 302
 303   /**
 304    * Put contents into a mysterious box.
 305    * @param {Object|Buffer} contents
 306    * @param {Number=} version
 307    * @param {Number=} flags
 308    * @returns {Promise<String>}
 309    */
 310   async pack(contents, version = this.bestVersion, flags = this.defaultFlags) {
 311     const { stats, timingsMs } = MysteryBox._newStats('pack');
 312
 313     if (!(version in this.versionParameters)) {
 314       throw new MysteryBoxError(`MysteryBox format version ${version} not supported`);
 315     }
 316     // eslint-disable-next-line security/detect-object-injection
 317     const v = this.versionParameters[version];
 318     stats.version = version;
 319
 320     const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags);
 321
 322     if (Buffer.isBuffer(contents)) {
 323       // Ensure payloadIsBuffer flag is set when contents are indeed a Buffer
 324       flags |= this.Flags.BufferPayload;
 325     } else {
 326       if (payloadIsBuffer) {
 327         // Flag is set, but contents are not a Buffer? Try to coerce contents.
 328         contents = Buffer.from(contents);
 329       } else {
 330         // Otherwise attempt to serialize the object
 331         contents = JSON.stringify(contents);
 332       }
 333     }
 334     stats.serializedBytes = Buffer.byteLength(contents);
 335
 336     const { buffer: versionBuffer, versionBytes } = MysteryBox._versionEncode(v.version);
 337     if (versionBytes !== v.versionBytes) {
 338       throw new MysteryBoxError('internal inconsistency, mismatched version byte length');
 339     }
 340
 341     const [iv, salt] = await Promise.all([
 342       v.ivBytes,
 343       v.saltBytes,
 344     ].map((b) => randomBytesAsync(b)));
 345
 346     timingsMs.preCompress = performance.now();
 347     let compressedContents;
 348     switch (compression) {
 349       case 0: // No compression requested
 350         compressedContents = contents;
 351         break;
 352       case this.Flags.Brotli:
 353         compressedContents = await brotliCompressAsync(contents);
 354         break;
 355       case this.Flags.Flate:
 356         compressedContents = await deflateRawAsync(contents);
 357         break;
 358     }
 359     stats.compressedBytes = Buffer.byteLength(compressedContents);
 360     // const compressionRatio = stats.compressedBytes / stats.serializedBytes;
 361     timingsMs.postCompress = timingsMs.preCrypt = performance.now();
 362
 363     let payload;
 364     if (stats.compressedBytes >= stats.serializedBytes) {
 365       // If compression is not beneficial enough, or detrimental, do not use
 366       flags = flags & ~compressionFlagsMask;
 367       payload = contents;
 368       stats.compressedBytes = undefined;
 369     } else {
 370       payload = compressedContents;
 371     }
 372
 373     const flagsBuffer = Buffer.alloc(v.flagsBytes);
 374     flagsBuffer.writeUInt8(flags, 0);
 375     stats.flagsRaw = flags;
 376     stats.flags = this._prettyFlags(flags);
 377
 378     // Authenticate all this data
 379     const aadBuffer = Buffer.concat([versionBuffer, flagsBuffer, iv, salt]);
 380
 381     // Always encrypt with first secret
 382     const secret = this.secrets[0];
 383     const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes);
 384     const cipher = crypto.createCipheriv(v.algorithm, key, iv, v.algOptions);
 385     cipher.setAAD(aadBuffer);
 386     const encrypted = cipher.update(payload);
 387     const final = cipher.final();
 388     const tag = cipher.getAuthTag();
 389
 390     const result = Buffer.concat([versionBuffer, flagsBuffer, iv, salt, tag, encrypted, final]).toString('base64url');
 391     timingsMs.end = timingsMs.postCrypt = performance.now();
 392
 393     this.emit('statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs), ...packageInfo });
 394
 395     return result;
 396   }
 397
 398
 399   /**
 400    * Take contents out of a mysterious box.
 401    * @param {String} box - Base64URL encoded payload
 402    * @returns {Promise<Object>}
 403    */
 404   async unpack(box) {
 405     const { stats, timingsMs } = MysteryBox._newStats('unpack');
 406
 407     if (!box) {
 408       throw new MysteryBoxError('nothing to unpack');
 409     }
 410
 411     const raw = Buffer.from(box, 'base64url');
 412     let offset = 0;
 413
 414     const { version, versionBytes } = MysteryBox._versionDecode(raw);
 415     if (!(version in this.versionParameters)) {
 416       throw new MysteryBoxError(`unsupported version (${version})`);
 417     }
 418     // eslint-disable-next-line security/detect-object-injection
 419     const v = this.versionParameters[version];
 420
 421     if (v.versionBytes !== versionBytes) {
 422       throw new MysteryBoxError('internal inconsistency, mismatched version byte length');
 423     }
 424     offset += v.versionBytes;
 425     stats.version = version;
 426
 427     const minBytes = v.versionBytes + v.flagsBytes + v.ivBytes + v.saltBytes + v.tagBytes;
 428     if (raw.length < minBytes) {
 429       throw new MysteryBoxError('not enough to unpack');
 430     }
 431
 432     const flags = raw.subarray(offset, offset + v.flagsBytes).readUInt8(0);
 433     offset += v.flagsBytes;
 434     stats.flagsRaw = flags;
 435
 436     const { compression, payloadIsBuffer } = MysteryBox._decodeFlags(flags);
 437
 438     const iv = raw.subarray(offset, offset + v.ivBytes);
 439     offset += v.ivBytes;
 440
 441     const salt = raw.subarray(offset, offset + v.saltBytes);
 442     offset += v.saltBytes;
 443
 444     const aad = raw.subarray(0, offset); // Everything up to here
 445
 446     const tag = raw.subarray(offset, offset + v.tagBytes);
 447     offset += v.tagBytes;
 448
 449     const encrypted = raw.subarray(offset);
 450
 451     timingsMs.preCrypt = performance.now();
 452
 453     let decrypted;
 454     let err;
 455     let success = false;
 456     for await (const secret of this.secrets) {
 457       const key = await MysteryBox._keyFromSecret(v.keyDeriver, secret, salt, v.keyBytes);
 458       const decipher = crypto.createDecipheriv(v.algorithm, key, iv, v.algOptions);
 459       decipher.setAAD(aad);
 460       decipher.setAuthTag(tag);
 461
 462       try {
 463         decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
 464         success = true;
 465         break;
 466       } catch (e) {
 467         err = e;
 468         continue;
 469       }
 470     }
 471     if (!success) {
 472       throw err;
 473     }
 474
 475     let payload;
 476     timingsMs.preCompress = timingsMs.postCrypt = performance.now();
 477     switch (compression) {
 478       case 0: // No compression
 479         payload = decrypted;
 480         break;
 481       case this.Flags.Brotli:
 482         payload = await brotliDecompressAsync(decrypted);
 483         break;
 484       case this.Flags.Flate:
 485         payload = await inflateRawAsync(decrypted);
 486         break;
 487     }
 488     timingsMs.end = timingsMs.postCompress = performance.now();
 489     stats.serializedBytes = payload.byteLength;
 490     if (compression) {
 491       stats.compressedBytes = decrypted.byteLength;
 492     }
 493
 494     if (!payloadIsBuffer) {
 495       payload = JSON.parse(payload.toString('utf8'));
 496     }
 497
 498     stats.flags = this._prettyFlags(flags);
 499     this.emit('statistics', { ...stats, ...MysteryBox._timingsLog(timingsMs), ...packageInfo });
 500
 501     return payload;
 502   }
 503
 504
 505   /**
 506    * Pretty-print flag values
 507    * @param {Number} flags
 508    * @returns {String}
 509    */
 510   _prettyFlags(flags) {
 511     const flagNames = Object.entries(this.Flags).reduce((acc, cur) => {
 512       const [flagName, flagValue] = cur;
 513       if ((flags & flagValue) === flagValue) {
 514         acc.push(flagName);
 515       }
 516       return acc;
 517     }, []);
 518     return `0x${flags.toString(16).padStart(2, '0')} [${flagNames.join(',')}]`;
 519   }
 520
 521
 522   /**
 523    * Everyone loves numbers.
 524    * @param {Object} timingsMs
 525    * @returns {Object}
 526    */
 527   static _timingsLog({ start, preCompress, postCompress, preCrypt, postCrypt, end }) {
 528     return {
 529       totalMs: end - start,
 530       compressMs: postCompress - preCompress,
 531       cryptMs: postCrypt - preCrypt,
 532     };
 533   }
 534
 535 }
 536
 537 module.exports = MysteryBox;