From 71efac9dcd7dc219cb83799391e7adc63cd4c662 Mon Sep 17 00:00:00 2001 From: Justin Wind Date: Mon, 4 Apr 2022 18:04:02 -0700 Subject: [PATCH] database migration 1.0.4, store topic fetch etag/last-modified, provide these when fetching topic contents. last-modified is specifically stored/requested as-is to simplify handling of odd topic servers. --- README.md | 2 +- documentation/media/postgres-er.svg | 6 +- documentation/media/sqlite-er.svg | 6 +- src/communication.js | 13 ++ src/db/base.js | 4 + src/db/postgres/index.js | 10 +- src/db/postgres/sql/schema/1.0.4/apply.sql | 10 ++ src/db/postgres/sql/schema/1.0.4/er.dot | 139 ++++++++++++++++++++ src/db/postgres/sql/schema/1.0.4/revert.sql | 10 ++ src/db/postgres/sql/topic-set-content.sql | 4 +- src/db/sqlite/index.js | 10 +- src/db/sqlite/sql/schema/1.0.4/apply.sql | 8 ++ src/db/sqlite/sql/schema/1.0.4/er.dot | 139 ++++++++++++++++++++ src/db/sqlite/sql/schema/1.0.4/revert.sql | 8 ++ src/db/sqlite/sql/topic-set-content.sql | 4 +- test/src/communication.js | 16 +++ 16 files changed, 380 insertions(+), 9 deletions(-) create mode 100644 src/db/postgres/sql/schema/1.0.4/apply.sql create mode 100644 src/db/postgres/sql/schema/1.0.4/er.dot create mode 100644 src/db/postgres/sql/schema/1.0.4/revert.sql create mode 100644 src/db/sqlite/sql/schema/1.0.4/apply.sql create mode 100644 src/db/sqlite/sql/schema/1.0.4/er.dot create mode 100644 src/db/sqlite/sql/schema/1.0.4/revert.sql diff --git a/README.md b/README.md index 5af15f0..33c8674 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ This implementation is built atop an in-house API framework, for Reasons. It wo - enum.js - invariants - errors.js - local Error types - link-helper.js - processes Link headers - - logger.js - a very simple logging class + - logger/ - adds service-specific data filters to our logging module - manager.js - process incoming requests - service.js - defines incoming endpoints, linking the API server framework to the manager methods - template/ - HTML content diff --git a/documentation/media/postgres-er.svg b/documentation/media/postgres-er.svg index b7db5b4..8610593 100644 --- a/documentation/media/postgres-er.svg +++ b/documentation/media/postgres-er.svg @@ -11,7 +11,7 @@ Websub Hub Entity-Relations Postgres -Schema 1.0.3 +Schema 1.0.4 topic @@ -52,6 +52,10 @@ content_hash content_type + +http_etag + +http_last_modified diff --git a/documentation/media/sqlite-er.svg b/documentation/media/sqlite-er.svg index e4a9c87..91752cb 100644 --- a/documentation/media/sqlite-er.svg +++ b/documentation/media/sqlite-er.svg @@ -11,7 +11,7 @@ Websub Hub Entity-Relations SQLite -Schema 1.0.3 +Schema 1.0.4 topic @@ -52,6 +52,10 @@ content_hash content_type + +http_etag + +http_last_modified diff --git a/src/communication.js b/src/communication.js index dc4d464..3b43678 100644 --- a/src/communication.js +++ b/src/communication.js @@ -196,6 +196,8 @@ class Communication { const acceptPreferred = [topic.contentType, acceptWildcard].filter((x) => x).join(', '); return Communication._axiosConfig('GET', topic.url, undefined, {}, { [Enum.Header.Accept]: acceptPreferred, + ...(topic.httpEtag && { [Enum.Header.IfNoneMatch]: topic.httpEtag }), + ...(topic.httpLastModified && { [Enum.Header.IfModifiedSince]: topic.httpLastModified }), }); } @@ -467,6 +469,7 @@ class Communication { switch (common.httpStatusCodeClass(response.status)) { case 2: + case 3: // Fall out of switch on success break; @@ -481,6 +484,12 @@ class Communication { return; } + if (response.status === 304) { + this.logger.info(_scope, 'content has not changed, per server', logInfoData); + await this.db.topicFetchComplete(dbCtx, topicId); + return; + } + const contentHash = Communication.contentHash(response.data, topic.contentHashAlgorithm); logInfoData.contentHash = contentHash; if (topic.contentHash === contentHash) { @@ -505,6 +514,8 @@ class Communication { } const contentType = response.headers[Enum.Header.ContentType.toLowerCase()]; + const httpETag = response.headers[Enum.Header.ETag.toLowerCase()]; + const httpLastModified = response.headers[Enum.Header.LastModified.toLowerCase()]; await this.db.transaction(dbCtx, async (txCtx) => { await this.db.topicSetContent(txCtx, { @@ -512,6 +523,8 @@ class Communication { content: Buffer.from(response.data), contentHash, ...(contentType && { contentType }), + ...(httpETag && { httpETag }), + ...(httpLastModified && { httpLastModified }), }); await this.db.topicFetchComplete(txCtx, topicId); diff --git a/src/db/base.js b/src/db/base.js index 3346f92..36cca60 100644 --- a/src/db/base.js +++ b/src/db/base.js @@ -173,6 +173,8 @@ class Database { this._ensureTypes(data, ['content'], ['string', 'buffer']); this._ensureTypes(data, ['contentHash'], ['string']); this._ensureTypes(data, ['contentType'], ['string', 'null', 'undefined']); + this._ensureTypes(data, ['eTag'], ['string', 'null', 'undefined']); + this._ensureTypes(data, ['lastModified'], ['string', 'null', 'undefined']); } @@ -614,6 +616,8 @@ class Database { * @param {String} data.content * @param {String} data.contentHash * @param {String=} data.contentType + * @param {String=} data.eTag + * @param {String=} data.lastModified */ async topicSetContent(dbCtx, data) { this._notImplemented('topicSetContent', arguments); diff --git a/src/db/postgres/index.js b/src/db/postgres/index.js index fc4ec8d..3451110 100644 --- a/src/db/postgres/index.js +++ b/src/db/postgres/index.js @@ -30,7 +30,7 @@ const schemaVersionsSupported = { max: { major: 1, minor: 0, - patch: 3, + patch: 4, }, }; @@ -900,6 +900,8 @@ class DatabasePostgres extends Database { const _scope = _fileScope('topicSetContent'); const topicSetContentData = { contentType: null, + httpETag: null, + httpLastModified: null, ...data, }; const logData = { @@ -916,7 +918,11 @@ class DatabasePostgres extends Database { if (result.rowCount != 1) { throw new DBErrors.UnexpectedResult('did not set topic content'); } - result = await dbCtx.result(this.statement.topicSetContentHistory, { topicId: data.topicId, contentHash: data.contentHash, contentSize: data.content.length }); + result = await dbCtx.result(this.statement.topicSetContentHistory, { + topicId: data.topicId, + contentHash: data.contentHash, + contentSize: data.content.length, + }); if (result.rowCount != 1) { throw new DBErrors.UnexpectedResult('did not set topic content history'); } diff --git a/src/db/postgres/sql/schema/1.0.4/apply.sql b/src/db/postgres/sql/schema/1.0.4/apply.sql new file mode 100644 index 0000000..514a6be --- /dev/null +++ b/src/db/postgres/sql/schema/1.0.4/apply.sql @@ -0,0 +1,10 @@ +BEGIN; + + ALTER TABLE topic + ADD COLUMN http_etag TEXT, + ADD COLUMN http_last_modified TEXT + ; + + INSERT INTO _meta_schema_version (major, minor, patch) VALUES (1, 0, 4); + +COMMIT; \ No newline at end of file diff --git a/src/db/postgres/sql/schema/1.0.4/er.dot b/src/db/postgres/sql/schema/1.0.4/er.dot new file mode 100644 index 0000000..4bd10b3 --- /dev/null +++ b/src/db/postgres/sql/schema/1.0.4/er.dot @@ -0,0 +1,139 @@ +digraph WebsubHubERD { + graph[ + rankdir=LR, + overlap=false, + splines=true, + label="Websub Hub Entity-Relations\nPostgres\nSchema 1.0.4", + labelloc="t", + fontsize=26, + ]; + // layout=neato; + node[shape=plain]; + edge[arrowhead=crow]; + + topic [label=< + + + + + + + + + + + + + + + + + + + + + +
TOPIC
id
created
url
lease_seconds_preferred
lease_seconds_min
lease_seconds_max
publisher_validation_url
content_hash_algorithm
is_active
is_deleted
last_publish
content_fetch_next_attempt
content_fetch_attempts_since_success
content_updated
content
content_hash
content_type
http_etag
http_last_modified
+ >]; + + topic_fetch_in_progress [label=< + + + + + + +
TOPIC_FETCH_IN_PROGRESS
id
claimant
claimed
claim_expires
+ >]; + topic:pk_id -> topic_fetch_in_progress:fk_id; + + topic_content_history [label=< + + + + + + +
TOPIC_CONTENT_HISTORY
topic_id
content_updated
content_size
content_hash
+ >]; + topic:pk_id -> topic_content_history:fk_topic_id; + + subscription [label=< + + + + + + + + + + + + + + + + +
SUBSCRIPTION
id
created
topic_id
callback
verified
expires
secret
signature_algorithm
http_remote_addr
http_from
content_delivered
latest_content_delivered
delivery_attempts_since_success
delivery_next_attempt
+ >]; + topic:pk_id -> subscription:fk_topic_id; + + subscription_delivery_in_progress [label=< + + + + + + +
SUBSCRIPTION_DELIVERY_IN_PROGRESS
id
claimant
claimed
claim_expires
+ >]; + subscription:pk_id -> subscription_delivery_in_progress:fk_id; + + verification [label=< + + + + + + + + + + + + + + + + + +
VERIFICATION
id
created
topic_id
callback
secret
signature_algorithm
http_remote_addr
http_from
mode
reason
lease_seconds
is_publisher_validated
request_id
attempts
next_attempt
+ >]; + topic:pk_id -> verification:fk_topic_id; + + verification_in_progress [label=< + + + + + + + + +
VERIFICATION_IN_PROGRESS
id
topic_id
callback
claimant
claimed
claim_expires
+ >]; + verification:pk_id -> verification_in_progress:fk_id; + topic:pk_id -> verification_in_progress:fk_topic_id; + + authentication [label=< + + + + + + +
AUTHENTICATION
created
last_authenticated
identifier
credential
+ >]; + +} \ No newline at end of file diff --git a/src/db/postgres/sql/schema/1.0.4/revert.sql b/src/db/postgres/sql/schema/1.0.4/revert.sql new file mode 100644 index 0000000..19c6e73 --- /dev/null +++ b/src/db/postgres/sql/schema/1.0.4/revert.sql @@ -0,0 +1,10 @@ +BEGIN; + + ALTER TABLE topic + DROP COLUMN http_etag, + DROP COLUMN http_last_modified + ; + + DELETE FROM _meta_schema_version WHERE major = 1 AND minor = 0 AND PATCH = 4; + +COMMIT; diff --git a/src/db/postgres/sql/topic-set-content.sql b/src/db/postgres/sql/topic-set-content.sql index c14258b..464410c 100644 --- a/src/db/postgres/sql/topic-set-content.sql +++ b/src/db/postgres/sql/topic-set-content.sql @@ -5,5 +5,7 @@ SET content_updated = now(), content = $(content), content_hash = $(contentHash), - content_type = $(contentType) + content_type = $(contentType), + http_etag = $(httpETag), + http_last_modified = $(httpLastModified) WHERE id = $(topicId) diff --git a/src/db/sqlite/index.js b/src/db/sqlite/index.js index 31471ba..56afa00 100644 --- a/src/db/sqlite/index.js +++ b/src/db/sqlite/index.js @@ -20,7 +20,7 @@ const schemaVersionsSupported = { max: { major: 1, minor: 0, - patch: 3, + patch: 4, }, }; @@ -865,6 +865,8 @@ class DatabaseSQLite extends Database { const _scope = _fileScope('topicSetContent'); const topicSetContentData = { contentType: null, + httpETag: null, + httpLastModified: null, ...data, }; const logData = { @@ -881,7 +883,11 @@ class DatabaseSQLite extends Database { if (result.changes != 1) { throw new DBErrors.UnexpectedResult('did not set topic content'); } - result = this.statement.topicSetContentHistory.run({ topicId: data.topicId, contentHash: data.contentHash, contentSize: data.content.length }); + result = this.statement.topicSetContentHistory.run({ + topicId: data.topicId, + contentHash: data.contentHash, + contentSize: data.content.length, + }); if (result.changes != 1) { throw new DBErrors.UnexpectedResult('did not set topic content history'); } diff --git a/src/db/sqlite/sql/schema/1.0.4/apply.sql b/src/db/sqlite/sql/schema/1.0.4/apply.sql new file mode 100644 index 0000000..bb4c297 --- /dev/null +++ b/src/db/sqlite/sql/schema/1.0.4/apply.sql @@ -0,0 +1,8 @@ +BEGIN; + + ALTER TABLE topic ADD COLUMN http_etag TEXT; + ALTER TABLE topic ADD COLUMN http_last_modified TEXT; + + INSERT INTO _meta_schema_version (major, minor, patch) VALUES (1, 0, 4); + +COMMIT; diff --git a/src/db/sqlite/sql/schema/1.0.4/er.dot b/src/db/sqlite/sql/schema/1.0.4/er.dot new file mode 100644 index 0000000..6d87839 --- /dev/null +++ b/src/db/sqlite/sql/schema/1.0.4/er.dot @@ -0,0 +1,139 @@ +digraph WebsubHubERD { + graph[ + rankdir=LR, + overlap=false, + splines=true, + label="Websub Hub Entity-Relations\nSQLite\nSchema 1.0.4", + labelloc="t", + fontsize=26, + ]; + // layout=neato; + node[shape=plain]; + edge[arrowhead=crow]; + + topic [label=< + + + + + + + + + + + + + + + + + + + + + +
TOPIC
id
created
url
lease_seconds_preferred
lease_seconds_min
lease_seconds_max
publisher_validation_url
content_hash_algorithm
is_active
is_deleted
last_publish
content_fetch_next_attempt
content_fetch_attempts_since_success
content_updated
content
content_hash
content_type
http_etag
http_last_modified
+ >]; + + topic_fetch_in_progress [label=< + + + + + + +
TOPIC_FETCH_IN_PROGRESS
id
claimant
claimed
claim_expires
+ >]; + topic:pk_id -> topic_fetch_in_progress:fk_id; + + topic_content_history [label=< + + + + + + +
TOPIC_CONTENT_HISTORY
topic_id
content_updated
content_size
content_hash
+ >]; + topic:pk_id -> topic_content_history:fk_topic_id; + + subscription [label=< + + + + + + + + + + + + + + + + +
SUBSCRIPTION
id
created
topic_id
callback
verified
expires
secret
signature_algorithm
http_remote_addr
http_from
content_delivered
latest_content_delivered
delivery_attempts_since_success
delivery_next_attempt
+ >]; + topic:pk_id -> subscription:fk_topic_id; + + subscription_delivery_in_progress [label=< + + + + + + +
SUBSCRIPTION_DELIVERY_IN_PROGRESS
id
claimant
claimed
claim_expires
+ >]; + subscription:pk_id -> subscription_delivery_in_progress:fk_id; + + verification [label=< + + + + + + + + + + + + + + + + + +
VERIFICATION
id
created
topic_id
callback
secret
signature_algorithm
http_remote_addr
http_from
mode
reason
lease_seconds
is_publisher_validated
request_id
attempts
next_attempt
+ >]; + topic:pk_id -> verification:fk_topic_id; + + verification_in_progress [label=< + + + + + + + + +
VERIFICATION_IN_PROGRESS
id
topic_id
callback
claimant
claimed
claim_expires
+ >]; + verification:pk_id -> verification_in_progress:fk_id; + topic:pk_id -> verification_in_progress:fk_topic_id; + + authentication [label=< + + + + + + +
AUTHENTICATION
created
last_authenticated
identifier
credential
+ >]; + +} \ No newline at end of file diff --git a/src/db/sqlite/sql/schema/1.0.4/revert.sql b/src/db/sqlite/sql/schema/1.0.4/revert.sql new file mode 100644 index 0000000..4cdf004 --- /dev/null +++ b/src/db/sqlite/sql/schema/1.0.4/revert.sql @@ -0,0 +1,8 @@ +BEGIN; + + ALTER TABLE topic DROP COLUMN http_etag; + ALTER TABLE topic DROP COLUMN http_last_modified; + + DELETE FROM _meta_schema_version WHERE major = 1 AND minor = 0 AND PATCH = 4; + +COMMIT; diff --git a/src/db/sqlite/sql/topic-set-content.sql b/src/db/sqlite/sql/topic-set-content.sql index 222d448..81480c9 100644 --- a/src/db/sqlite/sql/topic-set-content.sql +++ b/src/db/sqlite/sql/topic-set-content.sql @@ -5,5 +5,7 @@ SET content_updated = strftime('%s', 'now'), content = :content, content_hash = :contentHash, - content_type = :contentType + content_type = :contentType, + http_etag = :httpETag, + http_last_modified = :httpLastModified WHERE id = :topicId diff --git a/test/src/communication.js b/test/src/communication.js index 960a0f9..d5ebe54 100644 --- a/test/src/communication.js +++ b/test/src/communication.js @@ -552,6 +552,8 @@ describe('Communication', function () { headers: { 'content-type': 'text/plain', link: '; rel="hub"', + 'last-modified': 'Thu, 18 Nov 2021 20:34:35 GMT', + 'etag': '"9c104-1673e-5d1161636d742"', }, data: 'Jackdaws love my big sphinx of quartz.', }); @@ -631,6 +633,20 @@ describe('Communication', function () { assert(!communication.db.topicSetContent.called); }); + it('recognizes 304 response', async function () { + topic.httpLastModified = 'Thu, 18 Nov 2021 20:34:35 GMT'; + topic.httpEtag = '"9c104-1673e-5d1161636d742"'; + communication.db.topicGetById.resolves(topic); + communication.axios.resolves({ + status: 304, + }); + + await communication.topicFetchProcess(dbCtx, topicId, requestId); + + assert(communication.db.topicFetchComplete.called); + assert(!communication.db.topicSetContent.called); + }); + it('updates content', async function () { await communication.topicFetchProcess(dbCtx, topicId, requestId); -- 2.43.2