implement using ipv4 for fetch

Author: nleush

lib/fetch.js

@@ -3,6 +3,8 @@ import { h1NoCache, noCache, createUrl, AbortController, AbortError, FetchError
 import { serialize as serializeCookie, parse as parseCookie } from 'cookie';
 import log from '../logging.js';
 
+// TODO: review, use keepAliveNoCache, review keepAlive: true (default in many options).
+
 const fetchKeepAlive = noCache({
     h1: {
         keepAlive: true
@@ -20,13 +22,29 @@ const fetchH1KeepAlive = h1NoCache({
     rejectUnauthorized: false   // By default skip auth check for all.
 }).fetch;
 
+const fetchIPv4H1KeepAlive = h1NoCache({
+    h1: {
+        keepAlive: true,
+        family: 4
+    },
+    rejectUnauthorized: false   // By default skip auth check for all.
+}).fetch;
+
 const fetchAuthorized = noCache({
     h2: {
         enablePush: false
     }
 }).fetch;    // `rejectUnauthorized: true` - by `fetch` default.
+
 const fetchH1Authorized = h1NoCache().fetch;      // `rejectUnauthorized: true` - by `fetch` default.
 
+const fetchIPv4H1Authorized = h1NoCache({
+    h1: {
+        keepAlive: true,
+        family: 4
+    }
+}).fetch;      // `rejectUnauthorized: true` - by `fetch` default.
+
 const { fetch } = noCache({
     h2: {
         enablePush: false
@@ -38,7 +56,15 @@ const fetchH1 = h1NoCache({
     rejectUnauthorized: false   // By default skip auth check for all.
 }).fetch;
 
-function doFetch(fetch_func, h1_fetch_func, options) {
+const fetchIPv4H1 = h1NoCache({
+    h1: {
+        keepAlive: false,   // Default from `h1NoCache`.
+        family: 4
+    },
+    rejectUnauthorized: false   // By default skip auth check for all.
+}).fetch;
+
+function doFetch(fetch_func, h1_fetch_func, h1_ipv4_fetch_func, options) {
 
     const fetch_options = Object.assign({}, options);
 
@@ -56,7 +82,23 @@ function doFetch(fetch_func, h1_fetch_func, options) {
         abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
 
-    const a_fetch_func = options.disable_http2 ? h1_fetch_func: fetch_func;
+    var http2_used = false;
+    var a_fetch_func;
+    
+    if (options.use_ipv4) {
+
+        a_fetch_func = h1_ipv4_fetch_func;
+
+    } else if (options.disable_http2) {
+
+        a_fetch_func = h1_fetch_func;
+
+    } else {
+
+        a_fetch_func = fetch_func;
+        http2_used = true;
+    }
+
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)
             .then(response => {
@@ -67,11 +109,11 @@ function doFetch(fetch_func, h1_fetch_func, options) {
                 });
                 abortController.onResponse(stream);
 
-                if (response.status !== 200 && !options.disable_http2 && response.httpVersion === '2.0'
+                if (response.status !== 200 && http2_used && response.httpVersion === '2.0'
                     && CONFIG.DISABLE_HTTP2_CHECKS?.some(check => typeof check === 'function'
                                                                 && check(response.status, headers))) {
                     log('   -- doFetch check disabled h2', uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    resolve(doFetch(fetch_func, h1_fetch_func, h1_ipv4_fetch_func, Object.assign({}, options, {disable_http2: true})));
                 } else {
                     stream.status = response.status;
                     if (response.url && response.url !== uri) { // Set as final destination url if Fetch follows 301/302 re-directs
@@ -85,25 +127,25 @@ function doFetch(fetch_func, h1_fetch_func, options) {
             })
             .catch(error => {
                 clearTimeout(timeoutTimerId);
-                if (!options.disable_http2 && error.code && /^ERR_HTTP2/.test(error.code)) {
+                if (http2_used && error.code && /^ERR_HTTP2/.test(error.code)) {
 
                     log('   -- doFetch http2 error', error.code, uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    resolve(doFetch(fetch_func, h1_fetch_func, h1_ipv4_fetch_func, Object.assign({}, options, {disable_http2: true})));
 
-                } else if (!options.disable_http2 && error.code && error instanceof FetchError && error.code === 'ABORT_ERR') {
+                } else if (http2_used && error.code && error instanceof FetchError && error.code === 'ABORT_ERR') {
 
                     // Special case, when shared session request aborted by htmlparser logic.
                     /**
                      * https://polldaddy.com/poll/7451882/?s=twitter
                      * https://app.everviz.com/show/O0Cy7Dyt
                      */
                     log('   -- doFetch h2 aborted error', uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {disable_http2: true})));
+                    resolve(doFetch(fetch_func, h1_fetch_func, h1_ipv4_fetch_func, Object.assign({}, options, {disable_http2: true})));
 
                 } else if (!options.stopRecursion && CONFIG.ERRORS_TO_RETRY?.some(code => error.code?.indexOf(code) > -1)) {
 
                     log('   -- doFetch ECONNRESET retry', error.code, uri);
-                    resolve(doFetch(fetch_func, h1_fetch_func, Object.assign({}, options, {stopRecursion: true, disable_http2: true})));
+                    resolve(doFetch(fetch_func, h1_fetch_func, h1_ipv4_fetch_func, Object.assign({}, options, {stopRecursion: true, disable_http2: true})));
 
                 } else {
                     if (error instanceof AbortError) {
@@ -117,15 +159,15 @@ function doFetch(fetch_func, h1_fetch_func, options) {
 }
 
 export function fetchStreamKeepAlive(options) {
-    return doFetch(fetchKeepAlive, fetchH1KeepAlive, options);
+    return doFetch(fetchKeepAlive, fetchH1KeepAlive, fetchIPv4H1KeepAlive, options);
 }
 
 export function fetchStream(options) {
-    return doFetch(fetch, fetchH1, options);
+    return doFetch(fetch, fetchH1, fetchIPv4H1, options);
 };
 
 export function fetchStreamAuthorized(options) {
-    return doFetch(fetchAuthorized, fetchH1Authorized, options);
+    return doFetch(fetchAuthorized, fetchH1Authorized, fetchIPv4H1Authorized, options);
 };
 
 export function fetchData(options) {
@@ -144,7 +186,21 @@ export function fetchData(options) {
         abortController.abort();
     }, options.timeout || CONFIG.RESPONSE_TIMEOUT);
 
-    const a_fetch_func = options.disable_http2 ? fetchH1: fetch;
+    var a_fetch_func;
+    
+    if (options.use_ipv4) {
+
+        a_fetch_func = fetchIPv4H1;
+
+    } else if (options.disable_http2) {
+
+        a_fetch_func = fetchH1;
+
+    } else {
+
+        a_fetch_func = fetch;
+    }
+
     return new Promise((resolve, reject) => {
         a_fetch_func(uri, fetch_options)
             .then(response => {

lib/utils.js

@@ -124,9 +124,12 @@ export function prepareRequestOptions(request_options, options) {
             if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) {
                 options.maxRedirects = proxy.maxredirects;
             }
-            if (proxy && proxy.timeout > 0) {
+            if (proxy.timeout > 0) {
                 request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000;
             }
+            if (proxy.ipv4) {
+                request_options.use_ipv4 = true;
+            }
         }
     }
 
@@ -1393,4 +1396,4 @@ export function isBlocked(url, options, cb) {
                 return re.test(domain);
             }
         });
-}
+}