Merge branch 'v1.3-dev'

[websub-hub] / src / db / sqlite / index.js

diff --git a/src/db/sqlite/index.js b/src/db/sqlite/index.js
index fba4e7ca326f828a6f9698258d2ec4628fb7665e..6100d491a909547912cc6bdea61419706b4962e3 100644 (file)
--- a/src/db/sqlite/index.js
+++ b/src/db/sqlite/index.js
@@ -19,13 +19,15 @@ const schemaVersionsSupported = {
   },
   max: {
     major: 1,
   },
   max: {
     major: 1,

-    minor: 0,
-    patch: 1,
+    minor: 1,
+    patch: 0,

   },
 };
 
 // max of signed int64 (2^63 - 1), should be enough
 const EPOCH_FOREVER = BigInt('9223372036854775807');
   },
 };
 
 // max of signed int64 (2^63 - 1), should be enough
 const EPOCH_FOREVER = BigInt('9223372036854775807');

+const epochToDate = (epoch) => new Date(Number(epoch) * 1000);
+const dateToEpoch = (date) => Math.round(date.getTime() / 1000);

 
 class DatabaseSQLite extends Database {
   constructor(logger, options) {
 
 class DatabaseSQLite extends Database {
   constructor(logger, options) {


@@ -46,7 +48,7 @@ class DatabaseSQLite extends Database {
     this.db = new SQLite(dbFilename, sqliteOptions);
     this.schemaVersionsSupported = schemaVersionsSupported;
     this.changesSinceLastOptimize = BigInt(0);
     this.db = new SQLite(dbFilename, sqliteOptions);
     this.schemaVersionsSupported = schemaVersionsSupported;
     this.changesSinceLastOptimize = BigInt(0);

-    this.optimizeAfterChanges = options.db.connectionString.optimizeAfterChanges;
+    this.optimizeAfterChanges = options.db.optimizeAfterChanges;

     this.db.pragma('foreign_keys = on'); // Enforce consistency.
     this.db.pragma('journal_mode = WAL'); // Be faster, expect local filesystem.
     this.db.defaultSafeIntegers(true); // This probably isn't necessary, but by using these BigInts we keep weird floats out of the query logs.
     this.db.pragma('foreign_keys = on'); // Enforce consistency.
     this.db.pragma('journal_mode = WAL'); // Be faster, expect local filesystem.
     this.db.defaultSafeIntegers(true); // This probably isn't necessary, but by using these BigInts we keep weird floats out of the query logs.


@@ -68,7 +70,7 @@ class DatabaseSQLite extends Database {
     let metaExists = tableExists.get();
     if (metaExists === undefined) {
       const fPath = path.join(__dirname, 'sql', 'schema', 'init.sql');
     let metaExists = tableExists.get();
     if (metaExists === undefined) {
       const fPath = path.join(__dirname, 'sql', 'schema', 'init.sql');

-      // eslint-disable-next-line security/detect-non-literal-fs-filename
+       

       const fSql = fs.readFileSync(fPath, { encoding: 'utf8' });
       this.db.exec(fSql);
       metaExists = tableExists.get();
       const fSql = fs.readFileSync(fPath, { encoding: 'utf8' });
       this.db.exec(fSql);
       metaExists = tableExists.get();


@@ -85,10 +87,17 @@ class DatabaseSQLite extends Database {
     this.logger.debug(_scope, 'schema migrations wanted', { migrationsWanted });
     migrationsWanted.forEach((v) => {
       const fPath = path.join(__dirname, 'sql', 'schema', v, 'apply.sql');
     this.logger.debug(_scope, 'schema migrations wanted', { migrationsWanted });
     migrationsWanted.forEach((v) => {
       const fPath = path.join(__dirname, 'sql', 'schema', v, 'apply.sql');

-      // eslint-disable-next-line security/detect-non-literal-fs-filename
-      const fSql = fs.readFileSync(fPath, { encoding: 'utf8' });
-      this.logger.info(_scope, 'applying migration', { version: v });
-      this.db.exec(fSql);
+      try {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename
+        const fSql = fs.readFileSync(fPath, { encoding: 'utf8' });
+        this.logger.debug(_scope, 'applying migration', { version: v });
+        const results = this.db.exec(fSql);
+        this.logger.debug(_scope, 'migration results', { results });
+        this.logger.info(_scope, 'applied migration', { version: v });
+      } catch (e) {
+        this.logger.error(_scope, 'migration failed', { error: e, fPath, version: v });
+        throw e;
+      }

     });
   }
 
     });
   }
 


@@ -122,7 +131,7 @@ class DatabaseSQLite extends Database {
       };
     };
 
       };
     };
 

-    // eslint-disable-next-line security/detect-non-literal-fs-filename
+     

     for (const f of fs.readdirSync(sqlDir)) {
       const fPath = path.join(sqlDir, f);
       const { name: fName, ext: fExt } = path.parse(f);
     for (const f of fs.readdirSync(sqlDir)) {
       const fPath = path.join(sqlDir, f);
       const { name: fName, ext: fExt } = path.parse(f);


@@ -232,7 +241,7 @@ class DatabaseSQLite extends Database {
         'verification_in_progress',
         'subscription',
         'subscription_delivery_in_progress',
         'verification_in_progress',
         'subscription',
         'subscription_delivery_in_progress',

-      ].map((table) => {
+      ].forEach((table) => {

         const result = this.db.prepare(`DELETE FROM ${table}`).run();
         this.logger.debug(_fileScope('_purgeTables'), 'success', { table, result });
       });
         const result = this.db.prepare(`DELETE FROM ${table}`).run();
         this.logger.debug(_fileScope('_purgeTables'), 'success', { table, result });
       });


@@ -281,19 +290,56 @@ class DatabaseSQLite extends Database {
   }
 
 
   }
 
 

-  authenticationUpsert(dbCtx, identifier, credential) {
+  authenticationUpsert(dbCtx, identifier, credential, otpKey) {

     const _scope = _fileScope('authenticationUpsert');
     const scrubbedCredential = '*'.repeat((credential || '').length);
     const _scope = _fileScope('authenticationUpsert');
     const scrubbedCredential = '*'.repeat((credential || '').length);

-    this.logger.debug(_scope, 'called', { identifier, scrubbedCredential });
+    const scrubbedOTPKey = '*'.repeat((otpKey || '').length) || null;
+    this.logger.debug(_scope, 'called', { identifier, scrubbedCredential, scrubbedOTPKey });

 
     let result;
     try {
 
     let result;
     try {

-      result = this.statement.authenticationUpsert.run({ identifier, credential });
+      result = this.statement.authenticationUpsert.run({ identifier, credential, otpKey });

       if (result.changes != 1) {
         throw new DBErrors.UnexpectedResult('did not upsert authentication');
       }
     } catch (e) {
       if (result.changes != 1) {
         throw new DBErrors.UnexpectedResult('did not upsert authentication');
       }
     } catch (e) {

-      this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedCredential })
+      this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedCredential, scrubbedOTPKey });
+      throw e;
+    }
+  }
+
+
+  authenticationUpdateOTPKey(dbCtx, identifier, otpKey) {
+    const _scope = _fileScope('authenticationUpdateOTPKey');
+    const scrubbedOTPKey = '*'.repeat((otpKey || '').length) || null;
+    this.logger.debug(_scope, 'called', { identifier, scrubbedOTPKey });
+
+    let result;
+    try {
+      result = this.statement.authenticationUpdateOtpKey.run({ identifier, otpKey });
+      if (result.changes != 1) {
+        throw new DBErrors.UnexpectedResult('did not update authentication otp key');
+      }
+    } catch (e) {
+      this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedOTPKey });
+      throw e;
+    }
+  }
+
+
+  authenticationUpdateCredential(dbCtx, identifier, credential) {
+    const _scope = _fileScope('authenticationUpdateCredential');
+    const scrubbedCredential = '*'.repeat((credential || '').length);
+    this.logger.debug(_scope, 'called', { identifier, scrubbedCredential });
+
+    let result;
+    try {
+      result = this.statement.authenticationUpdateCredential.run({ identifier, credential });
+      if (result.changes != 1) {
+        throw new DBErrors.UnexpectedResult('did not update authentication credential');
+      }
+    } catch (e) {
+      this.logger.error(_scope, 'failed', { error: e, identifier, scrubbedCredential });

       throw e;
     }
   }
       throw e;
     }
   }


@@ -301,10 +347,10 @@ class DatabaseSQLite extends Database {
 
   /**
    * Converts engine subscription fields to native types.
 
   /**
    * Converts engine subscription fields to native types.

-   * @param {Object} data
+   * @param {object} data subscription data
+   * @returns {object} data

    */
   static _subscriptionDataToNative(data) {
    */
   static _subscriptionDataToNative(data) {

-    const epochToDate = (epoch) => new Date(Number(epoch) * 1000);

     if (data) {
       ['created', 'verified', 'expires', 'contentDelivered'].forEach((field) => {
         // eslint-disable-next-line security/detect-object-injection
     if (data) {
       ['created', 'verified', 'expires', 'contentDelivered'].forEach((field) => {
         // eslint-disable-next-line security/detect-object-injection


@@ -359,6 +405,21 @@ class DatabaseSQLite extends Database {
   }
 
 
   }
 
 

+  subscriptionDeleteExpired(dbCtx, topicId) {
+    const _scope = _fileScope('subscriptionDeleteExpired');
+    this.logger.debug(_scope, 'called', { topicId });
+
+    try {
+      const result = this.statement.subscriptionDeleteExpired.run({ topicId });
+      this.logger.debug(_scope, 'success', { topicId, deleted: result.changes });
+      return this._engineInfo(result);
+    } catch (e) {
+      this.logger.error(_scope, 'failed', { error: e, topicId });
+      throw e;
+    }
+  }
+
+

   subscriptionDeliveryClaim(dbCtx, wanted, claimTimeoutSeconds, claimant) {
     const _scope = _fileScope('subscriptionDeliveryClaim');
     this.logger.debug(_scope, 'called', { wanted, claimTimeoutSeconds, claimant });
   subscriptionDeliveryClaim(dbCtx, wanted, claimTimeoutSeconds, claimant) {
     const _scope = _fileScope('subscriptionDeliveryClaim');
     this.logger.debug(_scope, 'called', { wanted, claimTimeoutSeconds, claimant });


@@ -399,14 +460,15 @@ class DatabaseSQLite extends Database {
   }
 
 
   }
 
 

-  subscriptionDeliveryComplete(dbCtx, callback, topicId) {
+  subscriptionDeliveryComplete(dbCtx, callback, topicId, topicContentUpdated) {

     const _scope = _fileScope('subscriptionDeliveryComplete');
     const _scope = _fileScope('subscriptionDeliveryComplete');

-    this.logger.debug(_scope, 'called', { callback, topicId });
+    this.logger.debug(_scope, 'called', { callback, topicId, topicContentUpdated });

 
     let result;
     try {
       this.db.transaction(() => {
 
     let result;
     try {
       this.db.transaction(() => {

-        result = this.statement.subscriptionDeliverySuccess.run({ callback, topicId });
+        topicContentUpdated = dateToEpoch(topicContentUpdated);
+        result = this.statement.subscriptionDeliverySuccess.run({ callback, topicId, topicContentUpdated });

         if (result.changes != 1) {
           throw new DBErrors.UnexpectedResult('did not set subscription delivery success');
         }
         if (result.changes != 1) {
           throw new DBErrors.UnexpectedResult('did not set subscription delivery success');
         }


@@ -417,7 +479,7 @@ class DatabaseSQLite extends Database {
       })();
       return this._engineInfo(result);
     } catch (e) {
       })();
       return this._engineInfo(result);
     } catch (e) {

-      this.logger.error(_scope, 'failed', { error: e, callback, topicId });
+      this.logger.error(_scope, 'failed', { error: e, callback, topicId, topicContentUpdated });

       throw e;
     }
   }
       throw e;
     }
   }


@@ -534,7 +596,7 @@ class DatabaseSQLite extends Database {
       httpRemoteAddr: null,
       httpFrom: null,
       ...data,
       httpRemoteAddr: null,
       httpFrom: null,
       ...data,

-    }
+    };

     this._subscriptionUpsertDataValidate(subscriptionData);
 
     let result;
     this._subscriptionUpsertDataValidate(subscriptionData);
 
     let result;


@@ -680,10 +742,10 @@ class DatabaseSQLite extends Database {
 
   /**
    * Converts engine topic fields to native types.
 
   /**
    * Converts engine topic fields to native types.

-   * @param {Object} data
+   * @param {object} data topic
+   * @returns {object} topic data

    */
   static _topicDataToNative(data) {
    */
   static _topicDataToNative(data) {

-    const epochToDate = (epoch) => new Date(Number(epoch) * 1000);

     if (data) {
       data.isActive = !!data.isActive;
       data.isDeleted = !!data.isDeleted;
     if (data) {
       data.isActive = !!data.isActive;
       data.isDeleted = !!data.isDeleted;


@@ -736,7 +798,7 @@ class DatabaseSQLite extends Database {
   }
 
 
   }
 
 

-  topicGetByUrl(dbCtx, topicUrl) {
+  topicGetByUrl(dbCtx, topicUrl, applyDefaults = true) {

     const _scope = _fileScope('topicGetByUrl');
     this.logger.debug(_scope, 'called', { topicUrl });
 
     const _scope = _fileScope('topicGetByUrl');
     this.logger.debug(_scope, 'called', { topicUrl });
 


@@ -744,7 +806,10 @@ class DatabaseSQLite extends Database {
     try {
       topic = this.statement.topicGetByUrl.get({ topicUrl });
       DatabaseSQLite._topicDataToNative(topic);
     try {
       topic = this.statement.topicGetByUrl.get({ topicUrl });
       DatabaseSQLite._topicDataToNative(topic);

-      return this._topicDefaults(topic);
+      if (applyDefaults) {
+        topic = this._topicDefaults(topic);
+      }
+      return topic;

     } catch (e) {
       this.logger.error(_scope, 'failed', { error: e, topic, topicUrl });
       throw e;
     } catch (e) {
       this.logger.error(_scope, 'failed', { error: e, topic, topicUrl });
       throw e;


@@ -768,6 +833,50 @@ class DatabaseSQLite extends Database {
   }
 
 
   }
 
 

+  topicPendingDelete(dbCtx, topicId) {
+    const _scope = _fileScope('topicPendingDelete');
+    this.logger.debug(_scope, 'called', { topicId });
+
+    try {
+      this.db.transaction(() => {
+        const topic = this.statement.topicGetById.get({ topicId });
+        if (!topic.isDeleted) {
+          this.logger.debug(_scope, 'topic not set deleted, not deleting', { topicId });
+          return;
+        }
+
+        const { count: subscriberCount } = this.statement.subscriptionCountByTopicUrl.get({ topicUrl: topic.url });
+        if (subscriberCount) {
+          this.logger.debug(_scope, 'topic has subscribers, not deleting', { topicId, subscriberCount });
+          return;
+        }
+
+        const result = this.statement.topicDeleteById.run({ topicId });
+        if (result.changes !== 1) {
+          throw new DBErrors.UnexpectedResult('did not delete topic');
+        }
+      })();
+      this.logger.debug(_scope, 'success', { topicId });
+    } catch (e) {
+      this.logger.error(_scope, 'failed', { error: e, topicId });
+      throw e;
+    }
+  }
+
+
+  topicPublishHistory(dbCtx, topicId, days) {
+    const _scope = _fileScope('topicPublishHistory');
+    this.logger.debug(_scope, 'called', { topicId, days });
+
+    const events = this.statement.topicPublishHistory.all({ topicId, daysAgo: days });
+    const history = Array.from({ length: days }, () => 0);
+    // eslint-disable-next-line security/detect-object-injection
+    events.forEach(({ daysAgo, contentUpdates }) => history[daysAgo] = Number(contentUpdates));
+
+    return history;
+  }
+
+

   topicSet(dbCtx, data) {
     const _scope = _fileScope('topicSet');
     this.logger.debug(_scope, 'called', data);
   topicSet(dbCtx, data) {
     const _scope = _fileScope('topicSet');
     this.logger.debug(_scope, 'called', data);


@@ -799,6 +908,8 @@ class DatabaseSQLite extends Database {
     const _scope = _fileScope('topicSetContent');
     const topicSetContentData = {
       contentType: null,
     const _scope = _fileScope('topicSetContent');
     const topicSetContentData = {
       contentType: null,

+      httpETag: null,
+      httpLastModified: null,

       ...data,
     };
     const logData = {
       ...data,
     };
     const logData = {


@@ -815,6 +926,14 @@ class DatabaseSQLite extends Database {
       if (result.changes !=  1) {
         throw new DBErrors.UnexpectedResult('did not set topic content');
       }
       if (result.changes !=  1) {
         throw new DBErrors.UnexpectedResult('did not set topic content');
       }

+      result = this.statement.topicSetContentHistory.run({
+        topicId: data.topicId,
+        contentHash: data.contentHash,
+        contentSize: data.content.length,
+      });
+      if (result.changes != 1) {
+        throw new DBErrors.UnexpectedResult('did not set topic content history');
+      }

       return this._engineInfo(result);
     } catch (e) {
       this.logger.error(_scope, 'failed', { error: e, ...logData });
       return this._engineInfo(result);
     } catch (e) {
       this.logger.error(_scope, 'failed', { error: e, ...logData });


@@ -912,7 +1031,7 @@ class DatabaseSQLite extends Database {
 
   /**
    * Converts engine verification fields to native types.
 
   /**
    * Converts engine verification fields to native types.

-   * @param {Object} data
+   * @param {object} data verification

    */
   static _verificationDataToNative(data) {
     if (data) {
    */
   static _verificationDataToNative(data) {
     if (data) {


@@ -966,6 +1085,7 @@ class DatabaseSQLite extends Database {
 
   /**
    * Convert native verification fields to engine types.
 
   /**
    * Convert native verification fields to engine types.

+   * @param {object} data verification

    */
   static _verificationDataToEngine(data) {
     if (data) {
    */
   static _verificationDataToEngine(data) {
     if (data) {


@@ -1066,4 +1186,4 @@ class DatabaseSQLite extends Database {
 
 }
 
 
 }
 

-module.exports = DatabaseSQLite;
\ No newline at end of file
+module.exports = DatabaseSQLite;