Preserve MathJax tags (#958)

* Prevent MathJax nodes from being identified as 'unlikely candidates', and prevent <mjx-math> tags from being removed due to attribute aria-hidden="true"

* Revert changes to _isProbablyVisible() and isNodeVisible()

* Update test case to include the MathJax tags which are produced once client-side rendering is complete. The previous test case only used the static HTML received from the server.

Unfortunately, after htmltidy2 processes the page it is determined to be "unreaderable" though the appropriate JSDOM tests run and pass. Alternatively, if htmltidy2 is skipped, JSDOMParser produces a slew of errors.

Perhaps this will do for now...

* Adding support for file:// URLs. This is useful when the test case contains dynamic content as it allows the dev to save a copy of the rendered DOM to disk and use the resulting file as input to generate-testcase. Alternatively one could use JSDOM's {runScripts: "dangerously", resources: "usable"} options, but in my case these fell short and caused MathJax to crash due to missing localStorage implementation in JSDOM. Perhaps my approach will be useful to others...

* Use url fileURLToPath to handle file urls

---------

Co-authored-by: Gijs Kruitbosch <[email protected]>

Author: AlexTate

CONTRIBUTING.md

@@ -45,7 +45,7 @@ You can run it using:
     $ node test/generate-testcase.js slug https://example.com/article
 
 Replacing `slug` with the identifier the test should use, and providing a URL
-to an actual article on which the test should be based.
+to an actual article on which the test should be based. If your test case involves dynamic content, you can save the page to disk and pass the file as a `file://` URL.
 
 On macOS, you may need to make the `tidy` binary executable before that script will succeed. If you see an `EACCES` error when running that script, try:
 

Readability-readerable.js

@@ -24,7 +24,7 @@ var REGEXPS = {
   // Readability.js. Please keep both copies in sync.
   unlikelyCandidates:
     /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
-  okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
+  okMaybeItsACandidate: /and|article|body|column|content|main|mathjax|shadow/i,
 };
 
 function isNodeVisible(node) {

Readability.js

@@ -139,7 +139,8 @@ Readability.prototype = {
     // Readability-readerable.js. Please keep both copies in sync.
     unlikelyCandidates:
       /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
-    okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
+    okMaybeItsACandidate:
+      /and|article|body|column|content|main|mathjax|shadow/i,
 
     positive:
       /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,

test/generate-testcase.js

@@ -7,7 +7,7 @@ var fs = require("fs");
 var JSDOM = require("jsdom").JSDOM;
 var prettyPrint = require("./utils").prettyPrint;
 var http = require("http");
-var urlparse = require("url").parse;
+let { parse: urlparse, fileURLToPath } = require("url");
 var htmltidy = require("htmltidy2").tidy;
 
 var { Readability, isProbablyReaderable } = require("../index");
@@ -28,18 +28,9 @@ function generateTestcase(slug) {
       var sourceFile = path.join(destRoot, "source.html");
       fs.exists(sourceFile, function (exists) {
         if (exists) {
-          fs.readFile(
-            sourceFile,
-            { encoding: "utf-8" },
-            function (readFileErr, data) {
-              if (readFileErr) {
-                console.error("Source existed but couldn't be read?");
-                process.exit(1);
-                return;
-              }
-              onResponseReceived(null, data, destRoot);
-            }
-          );
+          fetchLocalSource(sourceFile, function (data) {
+            onResponseReceived(null, data, destRoot);
+          });
         } else {
           fetchSource(argURL, function (fetchErr, data) {
             onResponseReceived(fetchErr, data, destRoot);
@@ -60,29 +51,54 @@ function fetchSource(url, callbackFn) {
     process.exit(1);
     return;
   }
-  var client = http;
-  if (url.indexOf("https") == 0) {
-    client = require("https");
-  }
-  var options = urlparse(url);
-  options.headers = { "User-Agent": FFX_UA };
-
-  client.get(options, function (response) {
-    if (debug) {
-      console.log("STATUS:", response.statusCode);
-      console.log("HEADERS:", JSON.stringify(response.headers));
+  if (url.indexOf("http") == 0) {
+    var client = http;
+    if (url.indexOf("https") == 0) {
+      client = require("https");
     }
-    response.setEncoding("utf-8");
-    var rv = "";
-    response.on("data", function (chunk) {
-      rv += chunk;
-    });
-    response.on("end", function () {
+    var options = urlparse(url);
+    options.headers = { "User-Agent": FFX_UA };
+
+    client.get(options, function (response) {
       if (debug) {
-        console.log("End received");
+        console.log("STATUS:", response.statusCode);
+        console.log("HEADERS:", JSON.stringify(response.headers));
+      }
+      response.setEncoding("utf-8");
+      var rv = "";
+      response.on("data", function (chunk) {
+        rv += chunk;
+      });
+      response.on("end", function () {
+        if (debug) {
+          console.log("End received");
+        }
+        sanitizeSource(rv, callbackFn);
+      });
+    });
+  } else if (url.indexOf("file://") == 0) {
+    sourceFile = fileURLToPath(url);
+    fs.exists(sourceFile, function (exists) {
+      if (exists) {
+        fetchLocalSource(sourceFile, function (data) {
+          sanitizeSource(data, callbackFn);
+        });
+      } else {
+        console.error("File doesn't exist!");
+        process.exit(1);
       }
-      sanitizeSource(rv, callbackFn);
     });
+  }
+}
+
+function fetchLocalSource(sourceFile, callbackFn) {
+  fs.readFile(sourceFile, { encoding: "utf-8" }, function (readFileErr, data) {
+    if (readFileErr) {
+      console.error("Source existed but couldn't be read?");
+      process.exit(1);
+      return;
+    }
+    callbackFn(data);
   });
 }
 
@@ -94,6 +110,7 @@ function sanitizeSource(html, callbackFn) {
       "indent-spaces": 4,
       "numeric-entities": true,
       "output-xhtml": true,
+      "custom-tags": "blocklevel",
       wrap: 0,
     },
     callbackFn

test/test-pages/mathjax/expected-metadata.json

@@ -0,0 +1,10 @@
+{
+  "title": "MathJax v3 with MathML input and HTML output",
+  "byline": null,
+  "dir": null,
+  "lang": "en",
+  "excerpt": "When",
+  "siteName": null,
+  "publishedTime": null,
+  "readerable": false
+}