Resolve body URIs

.editorconfig

@@ -4,6 +4,7 @@ root = true
 [*]
 end_of_line = lf
 insert_final_newline = true
+trim_trailing_whitespace = true
 
 [{*.js,package.json,.travis.yml}]
 charset = utf-8

.eslintignore

@@ -1 +1,4 @@
+.vscode/
+.memsearch/
+node_modules/
 **/*.ts

.eslintrc.json

@@ -1,4 +1,5 @@
 {
+  "root": true,
   "env": {
     "node": true,
     "es6": true
@@ -8,6 +9,31 @@
   },
   "extends": "eslint:recommended",
   "rules": {
+    "arrow-spacing": [
+      "error",
+      {
+        "before": true,
+        "after": true
+      }
+    ],
+    "func-call-spacing": [
+      "error",
+      "never"
+    ],
+    "key-spacing": [
+      "error",
+      {
+        "beforeColon": false,
+        "afterColon": true
+      }
+    ],
+    "keyword-spacing": [
+      "error",
+      {
+        "before": true,
+        "after": true
+      }
+    ],
     "indent": [
       "error",
       2,
@@ -19,6 +45,10 @@
       "error",
       "unix"
     ],
+    "object-curly-spacing": [
+      "error",
+      "always"
+    ],
     "quotes": [
       "error",
       "single"
@@ -27,6 +57,17 @@
       "error",
       "always"
     ],
+    "space-before-blocks": [
+      "error",
+      "always"
+    ],
+    "space-before-function-paren": [
+      "error", {
+         "anonymous": "always",
+         "named": "always",
+         "asyncArrow": "always"
+      }
+    ],
     "no-cond-assign": [
       0
     ]

README.md

@@ -106,22 +106,23 @@ You can also check out this nice [working implementation](https://github.com/scr
 ### options
 
 - `normalize` - Set to `false` to override Feedparser's default behavior,
-  which is to parse feeds into an object that contains the generic properties
+  which is to both parse feeds into an object that contains the generic properties
   patterned after (although not identical to) the RSS 2.0 format, regardless
-  of the feed's format.
+  of the feed's format, as well as to resolve all relative urls, including those
+  embedded in HTML content fields.
 
 - `addmeta` - Set to `false` to override Feedparser's default behavior, which
   is to add the feed's `meta` information to each article.
 
 - `feedurl` - The url (string) of the feed. FeedParser is very good at
-  resolving relative urls in feeds. But some feeds use relative urls without
-  declaring the `xml:base` attribute any place in the feed. This is perfectly
-  valid, but we don't know know the feed's url before we start parsing the feed
-  and trying to resolve those relative urls. If we discover the feed's url, we
-  will go back and resolve the relative urls we've already seen, but this takes
-  a little time (not much). If you want to be sure we never have to re-resolve
-  relative urls (or if FeedParser is failing to properly resolve relative urls),
-  you should set the `feedurl` option. Otherwise, feel free to ignore this option.
+  resolving relative urls in feeds, including those embedded in HTML content 
+  fields. But some feeds use relative urls without declaring the `xml:base`
+  attribute any place in the feed. This is perfectly valid, but we don't know
+  the feed's url before we start parsing the feed and trying to resolve those
+  relative urls. If we discover the feed's url, we will go back and resolve the
+  relative urls we've already seen, but this takes a little time (not much).
+  If you want to be sure we can resolve all relative urls, you should set the
+  `feedurl` option.
 
 - `resume_saxerror` - Set to `false` to override Feedparser's default behavior, which
   is to silently handle them and then automatically resume parsing. In

bin/feedparser.js

@@ -36,7 +36,7 @@ var items = [];
 
 process.stdin.pipe(new FeedParser(argv))
   .on('error', console.error)
-  .on('readable', function() {
+  .on('readable', function () {
     var stream = this, item;
     while (item = stream.read()) {
       if (argv.group) {

examples/complete.js

@@ -8,15 +8,15 @@ var fetch = require('node-fetch')
   , FeedParser = require(__dirname+'/..')
   , iconv = require('iconv-lite');
 
-function get(feed) {
+function get (feed) {
   // Get a response stream
   fetch(feed, { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36', 'accept': 'text/html,application/xhtml+xml' }).then(function (res) {
 
     // Setup feedparser stream
     var feedparser = new FeedParser();
     feedparser.on('error', done);
     feedparser.on('end', done);
-    feedparser.on('readable', function() {
+    feedparser.on('readable', function () {
       var post;
       while (post = this.read()) {
         console.log(JSON.stringify(post, ' ', 4));
@@ -45,14 +45,14 @@ function maybeTranslate (res, charset) {
       // If we're using iconvStream, stream will be the output of iconvStream
       // otherwise it will remain the output of request
       res = res.pipe(iconvStream);
-    } catch(err) {
+    } catch (err) {
       res.emit('error', err);
     }
   }
   return res;
 }
 
-function getParams(str) {
+function getParams (str) {
   var params = str.split(';').reduce(function (params, param) {
     var parts = param.split('=').map(function (part) { return part.trim(); });
     if (parts.length === 2) {
@@ -63,7 +63,7 @@ function getParams(str) {
   return params;
 }
 
-function done(err) {
+function done (err) {
   if (err) {
     console.log(err, err.stack);
     return process.exit(1);

examples/simple.js

@@ -19,7 +19,7 @@ fs.createReadStream(feed)
   .on('meta', function (meta) {
     console.log('===== %s =====', meta.title);
   })
-  .on('readable', function() {
+  .on('readable', function () {
     var stream = this, item;
     while (item = stream.read()) {
       console.log('Got article: %s', item.title || item.description);

lib/namespaces.js → lib/constants.js

@@ -1,9 +1,10 @@
 /*
- * Default namespaces
- *
- * Lookup by URI
- */
-module.exports = {
+* Default namespaces
+*
+* Lookup by URI
+*/
+/* eslint-disable key-spacing */
+var NAMESPACES = {
   'http://www.w3.org/2005/Atom'                                    :'atom', // v1.0
   'http://purl.org/atom/ns#'                                       :'atom', // v0.3
   'http://www.w3.org/1999/02/22-rdf-syntax-ns#'                    :'rdf',
@@ -35,3 +36,49 @@ module.exports = {
   'http://www.w3.org/1999/xhtml'                                   :'xhtml',
   'http://www.w3.org/XML/1998/namespace'                           :'xml'
 };
+/* eslint-enable key-spacing */
+
+var HTML_URI_ATTRS = new Set([
+  'href',
+  'src',
+  'uri',
+  'srcset',
+  'cite',
+  'longdesc',
+  'action',
+  'background',
+  'data',
+  'poster'
+]);
+
+var HTML_TAGS = new Set([
+  'a', 'abbr', 'acronym', 'address', 'applet', 'area', 'article', 'aside', 'audio',
+  'b', 'base', 'basefont', 'bdi', 'bdo', 'big', 'blink', 'blockquote', 'body', 'br', 'button',
+  'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup',
+  'data', 'datalist', 'dd', 'del', 'details', 'dfn', 'dialog', 'dir', 'div', 'dl', 'dt',
+  'em', 'embed',
+  'fieldset', 'figcaption', 'figure', 'font', 'footer', 'form', 'frame', 'frameset',
+  'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'head', 'header', 'hgroup', 'hr', 'html',
+  'i', 'iframe', 'img', 'input', 'ins', 'isindex',
+  'kbd',
+  'label', 'legend', 'li', 'link', 'listing',
+  'main', 'map', 'mark', 'marquee', 'menu', 'menuitem', 'meta', 'meter', 'multicol',
+  'nav', 'nextid', 'nobr', 'noembed', 'noframes', 'noscript',
+  'object', 'ol', 'optgroup', 'option', 'output',
+  'p', 'param', 'picture', 'plaintext', 'pre', 'progress',
+  'q',
+  'rb', 'rp', 'rt', 'rtc', 'ruby',
+  's', 'samp', 'script', 'section', 'select', 'slot', 'small', 'source', 'spacer', 'span', 'strike', 'strong', 'style', 'sub', 'summary', 'sup',
+  'table', 'tbody', 'td', 'template', 'textarea', 'tfoot', 'th',
+  'thead', 'time', 'title', 'tr', 'track', 'tt',
+  'u', 'ul',
+  'var', 'video',
+  'wbr',
+  'xmp'
+]);
+
+module.exports = {
+  NAMESPACES,
+  HTML_URI_ATTRS,
+  HTML_TAGS
+};