update dependencies and devDependencies, fix issue with updated html parser not wanti...

[websub-hub] / src / link-helper.js

diff --git a/src/link-helper.js b/src/link-helper.js
index 1f3cd0992810f482717f0448781968afb0298723..93a947be58f284d63939000f5f342a3bf4fc5770 100644 (file)
--- a/src/link-helper.js
+++ b/src/link-helper.js
@@ -11,6 +11,7 @@ const Enum = require('./enum');
 const FeedParser = require('feedparser');
 const { Readable } = require('stream');
 const htmlparser2 = require('htmlparser2');
+const { Iconv } = require('iconv');
 
 const _fileScope = common.fileScope(__filename);
 
@@ -41,19 +42,34 @@ class LinkHelper {
     // Add Link headers first, as they take priority over link elements in body.
     const linkHeader = getHeader(headers, Enum.Header.Link);
     const links = [];
-    try {
-      links.push(...parseLinkHeader(linkHeader));
-    } catch (e) {
-      if (e instanceof ParseSyntaxError) {
-        this.logger.debug(_scope, 'failed to parse link header, bad syntax', { error: e, linkHeader });
-      } else {
-        this.logger.error(_scope, 'failed to parse link header', { error: e, linkHeader });
+    if (linkHeader) {
+      try {
+        links.push(...parseLinkHeader(linkHeader));
+      } catch (e) {
+        /* istanbul ignore else */
+        if (e instanceof ParseSyntaxError) {
+          this.logger.debug(_scope, 'failed to parse link header, bad syntax', { error: e, linkHeader });
+        } else {
+          this.logger.error(_scope, 'failed to parse link header', { error: e, linkHeader });
+        }
+      }
+    }
+
+    const contentType = LinkHelper.parseContentType(getHeader(headers, Enum.Header.ContentType));
+    const nonUTF8Charset = !/utf-*8/i.test(contentType.params.charset) && contentType.params.charset;
+    if (nonUTF8Charset) {
+      const iconv = new Iconv(nonUTF8Charset, 'utf-8//translit//ignore');
+      try {
+        body = iconv.convert(body).toString('utf8');
+      } catch (e) {
+        /* istanbul ignore next */
+        this.logger.error(_scope, 'iconv conversion error', { error: e, contentType, url });
+        // But try to carry on, anyhow.
       }
     }
 
-    const contentType = getHeader(headers, Enum.Header.ContentType);
     let bodyLinks = [];
-    switch (contentType) {
+    switch (contentType.mediaType) {
       case Enum.ContentType.ApplicationAtom:
       case Enum.ContentType.ApplicationRDF:
       case Enum.ContentType.ApplicationRSS:
@@ -75,10 +91,36 @@ class LinkHelper {
     // Fetch all hub relation targets from headers, resolving relative URIs.
     const hubs = LinkHelper.locateHubTargets(links).map((link) => this.absoluteURI(link, url));
 
+    this.logger.debug(_scope, 'valid hubs for url', { url, hubs });
+
     return hubs.includes(this.selfUrl);
   }
 
 
+  /**
+   * Convert a Content-Type string to normalized components.
+   * RFC7231 §3.1.1
+   * N.B. this non-parser implementation will not work if a parameter
+   * value for some reason includes a ; or = within a quoted-string.
+   * @param {String} contentTypeHeader
+   * @returns {Object} contentType
+   * @returns {String} contentType.mediaType
+   * @returns {Object} contentType.params
+   */
+  static parseContentType(contentTypeHeader) {
+    const [ mediaType, ...params ] = (contentTypeHeader || '').split(/ *; */);
+    return {
+      mediaType: mediaType.toLowerCase() || Enum.ContentType.ApplicationOctetStream,
+      params: params.reduce((obj, param) => {
+        const [field, value] = param.split('=');
+        const isQuoted = value.charAt(0) === '"' && value.charAt(value.length - 1) === '"';
+        obj[field.toLowerCase()] = isQuoted ? value.slice(1, value.length - 1) : value;
+        return obj;
+      }, {}),
+    };
+  }
+
+
   /**
    * Parse XML-ish feed content, extracting link elements into our own format.
    * @param {String} feedurl
@@ -106,7 +148,11 @@ class LinkHelper {
       });
       feedParser.on('meta', (meta) => {
         this.logger.debug(_scope, 'FeedParser meta', { meta });
-        const feedLinks = meta['atom:link'] || [];
+        let feedLinks = meta['atom:link'] || [];
+        if (!Array.isArray(feedLinks)) {
+          // Parsing RSS seems to return a single entry for this rather than a list.
+          feedLinks = [feedLinks];
+        }
         feedLinks
           .map((l) => l['@'])
           .forEach((l) => {