X-Git-Url: http://git.squeep.com/?a=blobdiff_plain;f=src%2Fdb%2Fsqlite%2Findex.js;h=a30c9b4ff00c673902708c3011e2d88dbc874f4d;hb=HEAD;hp=fba4e7ca326f828a6f9698258d2ec4628fb7665e;hpb=b2ddc9bc66b20975110561d7b3580ca1f5b9a7ce;p=websub-hub diff --git a/src/db/sqlite/index.js b/src/db/sqlite/index.js index fba4e7c..6100d49 100644 --- a/src/db/sqlite/index.js +++ b/src/db/sqlite/index.js @@ -19,13 +19,15 @@ const schemaVersionsSupported = { }, max: { major: 1, - minor: 0, - patch: 1, + minor: 1, + patch: 0, }, }; // max of signed int64 (2^63 - 1), should be enough const EPOCH_FOREVER = BigInt('9223372036854775807'); +const epochToDate = (epoch) => new Date(Number(epoch) * 1000); +const dateToEpoch = (date) => Math.round(date.getTime() / 1000); class DatabaseSQLite extends Database { constructor(logger, options) { @@ -46,7 +48,7 @@ class DatabaseSQLite extends Database { this.db = new SQLite(dbFilename, sqliteOptions); this.schemaVersionsSupported = schemaVersionsSupported; this.changesSinceLastOptimize = BigInt(0); - this.optimizeAfterChanges = options.db.connectionString.optimizeAfterChanges; + this.optimizeAfterChanges = options.db.optimizeAfterChanges; this.db.pragma('foreign_keys = on'); // Enforce consistency. this.db.pragma('journal_mode = WAL'); // Be faster, expect local filesystem. this.db.defaultSafeIntegers(true); // This probably isn't necessary, but by using these BigInts we keep weird floats out of the query logs. @@ -68,7 +70,7 @@ class DatabaseSQLite extends Database { let metaExists = tableExists.get(); if (metaExists === undefined) { const fPath = path.join(__dirname, 'sql', 'schema', 'init.sql'); - // eslint-disable-next-line security/detect-non-literal-fs-filename + const fSql = fs.readFileSync(fPath, { encoding: 'utf8' }); this.db.exec(fSql); metaExists = tableExists.get(); @@ -85,10 +87,17 @@ class DatabaseSQLite extends Database { this.logger.debug(_scope, 'schema migrations wanted', { migrationsWanted }); migrationsWanted.forEach((v) => { const fPath = path.join(__dirname, 'sql', 'schema', v, 'apply.sql'); - // eslint-disable-next-line security/detect-non-literal-fs-filename - const fSql = fs.readFileSync(fPath, { encoding: 'utf8' }); - this.logger.info(_scope, 'applying migration', { version: v }); - this.db.exec(fSql); + try { + // eslint-disable-next-line security/detect-non-literal-fs-filename + const fSql = fs.readFileSync(fPath, { encoding: 'utf8' }); + this.logger.debug(_scope, 'applying migration', { version: v }); + const results = this.db.exec(fSql); + this.logger.debug(_scope, 'migration results', { results }); + this.logger.info(_scope, 'applied migration', { version: v }); + } catch (e) { + this.logger.error(_scope, 'migration failed', { error: e, fPath, version: v }); + throw e; + } }); } @@ -122,7 +131,7 @@ class DatabaseSQLite extends Database { }; }; - // eslint-disable-next-line security/detect-non-literal-fs-filename + for (const f of fs.readdirSync(sqlDir)) { const fPath = path.join(sqlDir, f); const { name: fName, ext: fExt } = path.parse(f); @@ -232,7 +241,7 @@ class DatabaseSQLite extends Database { 'verification_in_progress', 'subscription', 'subscription_delivery_in_progress', - ].map((table) => { + ].forEach((table) => { const result = this.db.prepare(`DELETE FROM ${table}`).run(); this.logger.debug(_fileScope('_purgeTables'), 'success', { table, result }); }); @@ -281,19 +290,56 @@ class DatabaseSQLite extends Database { } - authenticationUpsert(dbCtx, identifier, credential) { + authenticationUpsert(dbCtx, identifier, credential, otpKey) { const _scope = _fileScope('authenticationUpsert'); const scrubbedCredential = '*'.repeat((credential || '').length); - this.logger.debug(_scope, 'called', { identifier, scrubbedCredential }); + const scrubbedOTPKey = '*'.repeat((otpKey || '').length) || null; + this.logger.debug(_scope, 'called', { identifier, scrubbedCredential, scrubbedOTPKey }); let result; try { - result = this.statement.authenticationUpsert.run({ identifier, credential }); + result = this.statement.authenticationUpsert.run({ identifier, credential, otpKey }); if (result.changes != 1) { throw new DBErrors.UnexpectedResult('did not upsert authentication'); } } catch (e) { - this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedCredential }) + this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedCredential, scrubbedOTPKey }); + throw e; + } + } + + + authenticationUpdateOTPKey(dbCtx, identifier, otpKey) { + const _scope = _fileScope('authenticationUpdateOTPKey'); + const scrubbedOTPKey = '*'.repeat((otpKey || '').length) || null; + this.logger.debug(_scope, 'called', { identifier, scrubbedOTPKey }); + + let result; + try { + result = this.statement.authenticationUpdateOtpKey.run({ identifier, otpKey }); + if (result.changes != 1) { + throw new DBErrors.UnexpectedResult('did not update authentication otp key'); + } + } catch (e) { + this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedOTPKey }); + throw e; + } + } + + + authenticationUpdateCredential(dbCtx, identifier, credential) { + const _scope = _fileScope('authenticationUpdateCredential'); + const scrubbedCredential = '*'.repeat((credential || '').length); + this.logger.debug(_scope, 'called', { identifier, scrubbedCredential }); + + let result; + try { + result = this.statement.authenticationUpdateCredential.run({ identifier, credential }); + if (result.changes != 1) { + throw new DBErrors.UnexpectedResult('did not update authentication credential'); + } + } catch (e) { + this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedCredential }); throw e; } } @@ -301,10 +347,10 @@ class DatabaseSQLite extends Database { /** * Converts engine subscription fields to native types. - * @param {Object} data + * @param {object} data subscription data + * @returns {object} data */ static _subscriptionDataToNative(data) { - const epochToDate = (epoch) => new Date(Number(epoch) * 1000); if (data) { ['created', 'verified', 'expires', 'contentDelivered'].forEach((field) => { // eslint-disable-next-line security/detect-object-injection @@ -359,6 +405,21 @@ class DatabaseSQLite extends Database { } + subscriptionDeleteExpired(dbCtx, topicId) { + const _scope = _fileScope('subscriptionDeleteExpired'); + this.logger.debug(_scope, 'called', { topicId }); + + try { + const result = this.statement.subscriptionDeleteExpired.run({ topicId }); + this.logger.debug(_scope, 'success', { topicId, deleted: result.changes }); + return this._engineInfo(result); + } catch (e) { + this.logger.error(_scope, 'failed', { error: e, topicId }); + throw e; + } + } + + subscriptionDeliveryClaim(dbCtx, wanted, claimTimeoutSeconds, claimant) { const _scope = _fileScope('subscriptionDeliveryClaim'); this.logger.debug(_scope, 'called', { wanted, claimTimeoutSeconds, claimant }); @@ -399,14 +460,15 @@ class DatabaseSQLite extends Database { } - subscriptionDeliveryComplete(dbCtx, callback, topicId) { + subscriptionDeliveryComplete(dbCtx, callback, topicId, topicContentUpdated) { const _scope = _fileScope('subscriptionDeliveryComplete'); - this.logger.debug(_scope, 'called', { callback, topicId }); + this.logger.debug(_scope, 'called', { callback, topicId, topicContentUpdated }); let result; try { this.db.transaction(() => { - result = this.statement.subscriptionDeliverySuccess.run({ callback, topicId }); + topicContentUpdated = dateToEpoch(topicContentUpdated); + result = this.statement.subscriptionDeliverySuccess.run({ callback, topicId, topicContentUpdated }); if (result.changes != 1) { throw new DBErrors.UnexpectedResult('did not set subscription delivery success'); } @@ -417,7 +479,7 @@ class DatabaseSQLite extends Database { })(); return this._engineInfo(result); } catch (e) { - this.logger.error(_scope, 'failed', { error: e, callback, topicId }); + this.logger.error(_scope, 'failed', { error: e, callback, topicId, topicContentUpdated }); throw e; } } @@ -534,7 +596,7 @@ class DatabaseSQLite extends Database { httpRemoteAddr: null, httpFrom: null, ...data, - } + }; this._subscriptionUpsertDataValidate(subscriptionData); let result; @@ -680,10 +742,10 @@ class DatabaseSQLite extends Database { /** * Converts engine topic fields to native types. - * @param {Object} data + * @param {object} data topic + * @returns {object} topic data */ static _topicDataToNative(data) { - const epochToDate = (epoch) => new Date(Number(epoch) * 1000); if (data) { data.isActive = !!data.isActive; data.isDeleted = !!data.isDeleted; @@ -736,7 +798,7 @@ class DatabaseSQLite extends Database { } - topicGetByUrl(dbCtx, topicUrl) { + topicGetByUrl(dbCtx, topicUrl, applyDefaults = true) { const _scope = _fileScope('topicGetByUrl'); this.logger.debug(_scope, 'called', { topicUrl }); @@ -744,7 +806,10 @@ class DatabaseSQLite extends Database { try { topic = this.statement.topicGetByUrl.get({ topicUrl }); DatabaseSQLite._topicDataToNative(topic); - return this._topicDefaults(topic); + if (applyDefaults) { + topic = this._topicDefaults(topic); + } + return topic; } catch (e) { this.logger.error(_scope, 'failed', { error: e, topic, topicUrl }); throw e; @@ -768,6 +833,50 @@ class DatabaseSQLite extends Database { } + topicPendingDelete(dbCtx, topicId) { + const _scope = _fileScope('topicPendingDelete'); + this.logger.debug(_scope, 'called', { topicId }); + + try { + this.db.transaction(() => { + const topic = this.statement.topicGetById.get({ topicId }); + if (!topic.isDeleted) { + this.logger.debug(_scope, 'topic not set deleted, not deleting', { topicId }); + return; + } + + const { count: subscriberCount } = this.statement.subscriptionCountByTopicUrl.get({ topicUrl: topic.url }); + if (subscriberCount) { + this.logger.debug(_scope, 'topic has subscribers, not deleting', { topicId, subscriberCount }); + return; + } + + const result = this.statement.topicDeleteById.run({ topicId }); + if (result.changes !== 1) { + throw new DBErrors.UnexpectedResult('did not delete topic'); + } + })(); + this.logger.debug(_scope, 'success', { topicId }); + } catch (e) { + this.logger.error(_scope, 'failed', { error: e, topicId }); + throw e; + } + } + + + topicPublishHistory(dbCtx, topicId, days) { + const _scope = _fileScope('topicPublishHistory'); + this.logger.debug(_scope, 'called', { topicId, days }); + + const events = this.statement.topicPublishHistory.all({ topicId, daysAgo: days }); + const history = Array.from({ length: days }, () => 0); + // eslint-disable-next-line security/detect-object-injection + events.forEach(({ daysAgo, contentUpdates }) => history[daysAgo] = Number(contentUpdates)); + + return history; + } + + topicSet(dbCtx, data) { const _scope = _fileScope('topicSet'); this.logger.debug(_scope, 'called', data); @@ -799,6 +908,8 @@ class DatabaseSQLite extends Database { const _scope = _fileScope('topicSetContent'); const topicSetContentData = { contentType: null, + httpETag: null, + httpLastModified: null, ...data, }; const logData = { @@ -815,6 +926,14 @@ class DatabaseSQLite extends Database { if (result.changes != 1) { throw new DBErrors.UnexpectedResult('did not set topic content'); } + result = this.statement.topicSetContentHistory.run({ + topicId: data.topicId, + contentHash: data.contentHash, + contentSize: data.content.length, + }); + if (result.changes != 1) { + throw new DBErrors.UnexpectedResult('did not set topic content history'); + } return this._engineInfo(result); } catch (e) { this.logger.error(_scope, 'failed', { error: e, ...logData }); @@ -912,7 +1031,7 @@ class DatabaseSQLite extends Database { /** * Converts engine verification fields to native types. - * @param {Object} data + * @param {object} data verification */ static _verificationDataToNative(data) { if (data) { @@ -966,6 +1085,7 @@ class DatabaseSQLite extends Database { /** * Convert native verification fields to engine types. + * @param {object} data verification */ static _verificationDataToEngine(data) { if (data) { @@ -1066,4 +1186,4 @@ class DatabaseSQLite extends Database { } -module.exports = DatabaseSQLite; \ No newline at end of file +module.exports = DatabaseSQLite;