From 9da684fbf838ab94f447843cb10087ed8a02ad77 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 3 Apr 2025 15:37:36 +0800 Subject: [PATCH 01/12] Add support for proxy --- src/config.ts | 18 ++++++++++++++++++ src/utils.ts | 43 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index 2a17600..09c2575 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,8 +1,26 @@ export type Config = { api_key: string | null; timeout: number; + http_proxy?: string; + https_proxy?: string; + no_proxy?: string; }; + +function getEnvVar(name: string): string | undefined { + if (typeof Deno !== "undefined") { + return Deno.env.get(name); + // @ts-ignore: Node.js process + } else if (typeof process !== "undefined") { + // @ts-ignore: Node.js process + return process.env[name]; + } + return undefined; +} + export const config: Config = { api_key: null, timeout: 60000, + http_proxy: getEnvVar("HTTP_PROXY") || getEnvVar("http_proxy"), + https_proxy: getEnvVar("HTTPS_PROXY") || getEnvVar("https_proxy"), + no_proxy: getEnvVar("NO_PROXY") || getEnvVar("no_proxy"), }; diff --git a/src/utils.ts b/src/utils.ts index a05fa05..7f4de7c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,7 +1,11 @@ import { version } from "../version.ts"; import https from "node:https"; +import http from "node:http"; import qs from "node:querystring"; +import { HttpsProxyAgent } from "npm:https-proxy-agent"; import { RequestTimeoutError } from "./errors.ts"; +import { config } from "./config.ts"; +import { Buffer } from "node:buffer"; /** * This `_internals` object is needed to support stubbing/spying of @@ -60,18 +64,30 @@ export function execute( ...parameters, source: getSource(), }); + + // Check if we should use a proxy + const urlObj = new URL(url); + const shouldUseProxy = !config.no_proxy?.split(",").some((domain) => + urlObj.hostname.endsWith(domain.trim()) + ); + + const proxyUrl = shouldUseProxy + ? (urlObj.protocol === "https:" ? config.https_proxy : config.http_proxy) + : undefined; + return new Promise((resolve, reject) => { let timer: number; - const req = https.get(url, (resp) => { + + const handleResponse = (resp: http.IncomingMessage) => { resp.setEncoding("utf8"); let data = ""; - // A chunk of data has been recieved. - resp.on("data", (chunk) => { + // A chunk of data has been received + resp.on("data", (chunk: Buffer) => { data += chunk; }); - // The whole response has been received. Print out the result. + // The whole response has been received resp.on("end", () => { try { if (resp.statusCode == 200) { @@ -85,10 +101,25 @@ export function execute( if (timer) clearTimeout(timer); } }); - }).on("error", (err) => { + }; + + const handleError = (err: Error) => { reject(err); if (timer) clearTimeout(timer); - }); + }; + + const options: https.RequestOptions = { + timeout: timeout > 0 ? timeout : undefined, + }; + + if (proxyUrl) { + options.agent = new HttpsProxyAgent(proxyUrl); + } + + const req = https.get(url, options, handleResponse).on( + "error", + handleError, + ); if (timeout > 0) { timer = setTimeout(() => { From 39ca5c3862fc10678fc90c0e4a676a45cc1311ac Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 3 Apr 2025 15:45:32 +0800 Subject: [PATCH 02/12] Update workflows --- .github/workflows/build.yml | 28 ++++++++++++++-------------- .github/workflows/release.yml | 12 ++++++------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83b4b27..6cf702c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,15 +5,15 @@ on: [push] jobs: build: name: "Deno tests and build npm files" - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v3 - name: Setup Deno - uses: denoland/setup-deno@v1.1.1 + uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Check formatting run: deno fmt --check @@ -27,9 +27,9 @@ jobs: run: deno task test:ci - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: '18.x' # Build files using a fixed node version + node-version: '22.x' # Build files using a fixed node version registry-url: 'https://registry.npmjs.org' - name: Build npm files @@ -39,7 +39,7 @@ jobs: run: zip npm.zip ./npm -r - name: Upload build files for smoke tests - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: npm path: npm.zip @@ -48,7 +48,7 @@ jobs: smoke-tests-commonjs: name: "Smoke tests (CommonJS)" needs: build - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] @@ -63,16 +63,16 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} registry-url: 'https://registry.npmjs.org' - name: Download build files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: npm @@ -90,22 +90,22 @@ jobs: smoke-tests-esm: name: "Smoke tests (ESM)" needs: build - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: node-version: [14.x, 15.x, 16.x, 17.x, 18.x, 19.x] steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} registry-url: 'https://registry.npmjs.org' - name: Download build files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: npm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e35e4e..4dc5e60 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,15 +9,15 @@ on: jobs: release: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Deno - uses: denoland/setup-deno@v1.1.1 + uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Check formatting run: deno fmt --check @@ -31,9 +31,9 @@ jobs: run: deno task test:ci - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: '18.x' + node-version: '22.x' registry-url: 'https://registry.npmjs.org' - name: Build npm files From b74eb37d4036d065325ca83dce4d5cd26c6f1ac1 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 3 Apr 2025 15:51:22 +0800 Subject: [PATCH 03/12] Upgrade dnt --- scripts/build_npm.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 4b92248..a88a0de 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -1,4 +1,4 @@ -import { build, emptyDir } from "https://deno.land/x/dnt@0.37.0/mod.ts"; +import { build, emptyDir } from "https://deno.land/x/dnt@0.40.0/mod.ts"; import { version } from "../version.ts"; await emptyDir("./npm"); From 776d5bea7272b8f0d57037bb638e2e599902e2f5 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 3 Apr 2025 15:57:51 +0800 Subject: [PATCH 04/12] Fix lint --- examples/node/js_node_14_up/basic_example.js | 1 + examples/node/js_node_14_up/pagination_example.js | 1 + examples/node/js_node_7_up/basic_example.js | 1 + examples/node/js_node_7_up/pagination_example.js | 1 + smoke_tests/commonjs/commonjs.js | 1 + smoke_tests/esm/esm.js | 1 + src/config.ts | 4 ++-- src/utils.ts | 3 +-- 8 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/node/js_node_14_up/basic_example.js b/examples/node/js_node_14_up/basic_example.js index 41c36cd..9c81fc7 100644 --- a/examples/node/js_node_14_up/basic_example.js +++ b/examples/node/js_node_14_up/basic_example.js @@ -7,6 +7,7 @@ */ import * as Dotenv from "dotenv"; +import process from "process"; import { config, getJson } from "serpapi"; Dotenv.config(); diff --git a/examples/node/js_node_14_up/pagination_example.js b/examples/node/js_node_14_up/pagination_example.js index 07c705a..15619a5 100644 --- a/examples/node/js_node_14_up/pagination_example.js +++ b/examples/node/js_node_14_up/pagination_example.js @@ -7,6 +7,7 @@ */ import * as Dotenv from "dotenv"; +import process from "process"; import { config, getJson } from "serpapi"; Dotenv.config(); diff --git a/examples/node/js_node_7_up/basic_example.js b/examples/node/js_node_7_up/basic_example.js index efe56c2..e03dd61 100644 --- a/examples/node/js_node_7_up/basic_example.js +++ b/examples/node/js_node_7_up/basic_example.js @@ -3,6 +3,7 @@ */ const Dotenv = require("dotenv"); +const process = require("process"); const { config, getJson } = require("serpapi"); Dotenv.config(); diff --git a/examples/node/js_node_7_up/pagination_example.js b/examples/node/js_node_7_up/pagination_example.js index fdaebf8..691b13d 100644 --- a/examples/node/js_node_7_up/pagination_example.js +++ b/examples/node/js_node_7_up/pagination_example.js @@ -6,6 +6,7 @@ const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); const url = require("url"); const qs = require("querystring"); +const process = require("process"); Dotenv.config(); config.api_key = process.env.API_KEY; diff --git a/smoke_tests/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js index 46c4ff2..fde3f4e 100644 --- a/smoke_tests/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -3,6 +3,7 @@ */ const Dotenv = require("dotenv"); +const process = require("process"); const { config, getJson, diff --git a/smoke_tests/esm/esm.js b/smoke_tests/esm/esm.js index ad4bee2..080eb85 100644 --- a/smoke_tests/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -7,6 +7,7 @@ */ import Dotenv from "dotenv"; +import process from "process"; import { config, getAccount, diff --git a/src/config.ts b/src/config.ts index 09c2575..fc764a1 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,5 @@ +import process from "node:process"; + export type Config = { api_key: string | null; timeout: number; @@ -9,9 +11,7 @@ export type Config = { function getEnvVar(name: string): string | undefined { if (typeof Deno !== "undefined") { return Deno.env.get(name); - // @ts-ignore: Node.js process } else if (typeof process !== "undefined") { - // @ts-ignore: Node.js process return process.env[name]; } return undefined; diff --git a/src/utils.ts b/src/utils.ts index 7f4de7c..68be986 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -6,6 +6,7 @@ import { HttpsProxyAgent } from "npm:https-proxy-agent"; import { RequestTimeoutError } from "./errors.ts"; import { config } from "./config.ts"; import { Buffer } from "node:buffer"; +import process from "node:process"; /** * This `_internals` object is needed to support stubbing/spying of @@ -31,9 +32,7 @@ export function getSource() { if (denoVersion) { return `deno@${denoVersion},${moduleSource}`; } - // @ts-ignore: scope of nodejs } else if (typeof process == "object") { - // @ts-ignore: scope of nodejs const nodeVersion = process.versions?.node; if (nodeVersion) { return `nodejs@${nodeVersion},${moduleSource}`; From 86cc3bb46a727cc34e43205fcd9c8171cfab044f Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 3 Apr 2025 16:01:47 +0800 Subject: [PATCH 05/12] Fix deno fmt --- .github/workflows/build.yml | 24 +++++++++++++++++++----- .github/workflows/release.yml | 6 +++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cf702c..392b97f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,8 +29,8 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: '22.x' # Build files using a fixed node version - registry-url: 'https://registry.npmjs.org' + node-version: "22.x" # Build files using a fixed node version + registry-url: "https://registry.npmjs.org" - name: Build npm files run: deno task npm @@ -51,7 +51,21 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + node-version: [ + 7.x, + 8.x, + 9.x, + 10.x, + 11.x, + 12.x, + 13.x, + 14.x, + 15.x, + 16.x, + 17.x, + 18.x, + 19.x, + ] include: - command: test - command: test:use-openssl-ca @@ -69,7 +83,7 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" - name: Download build files uses: actions/download-artifact@v4 @@ -102,7 +116,7 @@ jobs: uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" - name: Download build files uses: actions/download-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4dc5e60..1297b81 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ on: branches: - master paths: - - 'version.ts' + - "version.ts" jobs: release: @@ -33,8 +33,8 @@ jobs: - name: Setup Node uses: actions/setup-node@v4 with: - node-version: '22.x' - registry-url: 'https://registry.npmjs.org' + node-version: "22.x" + registry-url: "https://registry.npmjs.org" - name: Build npm files run: deno task npm From e986287431bdaad6ce78903ee6b886982428d142 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 3 Apr 2025 16:01:52 +0800 Subject: [PATCH 06/12] Fix deno warning --- deno.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/deno.json b/deno.json index 1a6d5f5..03d6ad8 100644 --- a/deno.json +++ b/deno.json @@ -8,9 +8,7 @@ "npm": "deno run -A scripts/build_npm.ts" }, "fmt": { - "files": { - "exclude": ["npm/", "examples/node", "smoke_tests/"] - } + "exclude": ["npm/", "examples/node", "smoke_tests/"] }, "lint": { "files": { From 94e34132643dd6b88731e41687064d57b546f8a0 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sat, 5 Apr 2025 17:47:13 +0800 Subject: [PATCH 07/12] Remove HttpsProxyAgent, allow users to pass http.RequestOptions instead --- src/config.ts | 18 ++---------------- src/utils.ts | 28 ++++++---------------------- 2 files changed, 8 insertions(+), 38 deletions(-) diff --git a/src/config.ts b/src/config.ts index fc764a1..386b7ea 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,26 +1,12 @@ -import process from "node:process"; +import http from "node:http"; export type Config = { api_key: string | null; timeout: number; - http_proxy?: string; - https_proxy?: string; - no_proxy?: string; + requestOptions?: http.RequestOptions; }; -function getEnvVar(name: string): string | undefined { - if (typeof Deno !== "undefined") { - return Deno.env.get(name); - } else if (typeof process !== "undefined") { - return process.env[name]; - } - return undefined; -} - export const config: Config = { api_key: null, timeout: 60000, - http_proxy: getEnvVar("HTTP_PROXY") || getEnvVar("http_proxy"), - https_proxy: getEnvVar("HTTPS_PROXY") || getEnvVar("https_proxy"), - no_proxy: getEnvVar("NO_PROXY") || getEnvVar("no_proxy"), }; diff --git a/src/utils.ts b/src/utils.ts index 68be986..98737f4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,7 +2,6 @@ import { version } from "../version.ts"; import https from "node:https"; import http from "node:http"; import qs from "node:querystring"; -import { HttpsProxyAgent } from "npm:https-proxy-agent"; import { RequestTimeoutError } from "./errors.ts"; import { config } from "./config.ts"; import { Buffer } from "node:buffer"; @@ -64,16 +63,6 @@ export function execute( source: getSource(), }); - // Check if we should use a proxy - const urlObj = new URL(url); - const shouldUseProxy = !config.no_proxy?.split(",").some((domain) => - urlObj.hostname.endsWith(domain.trim()) - ); - - const proxyUrl = shouldUseProxy - ? (urlObj.protocol === "https:" ? config.https_proxy : config.http_proxy) - : undefined; - return new Promise((resolve, reject) => { let timer: number; @@ -107,18 +96,13 @@ export function execute( if (timer) clearTimeout(timer); }; - const options: https.RequestOptions = { - timeout: timeout > 0 ? timeout : undefined, - }; - - if (proxyUrl) { - options.agent = new HttpsProxyAgent(proxyUrl); - } + const options = (parameters.requestOptions as http.RequestOptions) || + config.requestOptions || + {}; - const req = https.get(url, options, handleResponse).on( - "error", - handleError, - ); + const req = https + .get(url, options, handleResponse) + .on("error", handleError); if (timeout > 0) { timer = setTimeout(() => { From f6c53ea73e4f4dfe69156a412edb4b7138197087 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sat, 5 Apr 2025 18:08:48 +0800 Subject: [PATCH 08/12] Backwards compatibility for Node <= 9 --- src/utils.ts | 35 +- tests/serpapi_test.ts | 1098 ++++++++++++++++++++++------------------- tests/utils_test.ts | 103 ++-- 3 files changed, 676 insertions(+), 560 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 98737f4..ff3f88f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -16,12 +16,15 @@ import process from "node:process"; */ export const _internals = { execute: execute, - getBaseUrl: getBaseUrl, + getHostnameAndPort: getHostnameAndPort, }; /** Facilitates stubbing in tests, e.g. localhost as the base url */ -function getBaseUrl() { - return "https://serpapi.com"; +function getHostnameAndPort() { + return { + hostname: "serpapi.com", + port: 443, + }; } export function getSource() { @@ -40,17 +43,27 @@ export function getSource() { return `nodejs,${moduleSource}`; } -export function buildUrl( +export function buildRequestOptions( path: string, parameters: qs.ParsedUrlQueryInput, -): string { +): http.RequestOptions { const clonedParams = { ...parameters }; for (const k in clonedParams) { if (clonedParams[k] === undefined) { delete clonedParams[k]; } } - return `${_internals.getBaseUrl()}${path}?${qs.stringify(clonedParams)}`; + const base = { + ..._internals.getHostnameAndPort(), + path: `${path}?${qs.stringify(clonedParams)}`, + method: "GET", + }; + + return { + ...base, + ...(parameters.requestOptions as http.RequestOptions), + ...config.requestOptions, + }; } export function execute( @@ -58,7 +71,7 @@ export function execute( parameters: qs.ParsedUrlQueryInput, timeout: number, ): Promise { - const url = buildUrl(path, { + const options = buildRequestOptions(path, { ...parameters, source: getSource(), }); @@ -96,13 +109,7 @@ export function execute( if (timer) clearTimeout(timer); }; - const options = (parameters.requestOptions as http.RequestOptions) || - config.requestOptions || - {}; - - const req = https - .get(url, options, handleResponse) - .on("error", handleError); + const req = https.get(options, handleResponse).on("error", handleError); if (timeout > 0) { timer = setTimeout(() => { diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index ea58049..e22143b 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -40,565 +40,637 @@ import { loadSync({ export: true }); const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("getAccount", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with no api_key", () => { - assertRejects( - async () => await getAccount({ api_key: "" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getAccount({}), - MissingApiKeyError, - ); - assertRejects( - async () => await getAccount(), - MissingApiKeyError, - ); - }); +const BASE_OPTIONS = Deno.env.get("ENV_TYPE") === "local" + ? { + hostname: "localhost", + port: 3000, + } + : { + hostname: "serpapi.com", + port: 443, + }; + +describe( + "getAccount", + { + sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - it("with invalid timeout", { - ignore: !HAS_API_KEY, - }, () => { - config.api_key = SERPAPI_TEST_KEY; - assertRejects( - async () => await getAccount({ timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getAccount({ timeout: -10 }), - InvalidTimeoutError, - ); - }); + afterEach(() => { + config.api_key = null; + }); - it("async/await", { - ignore: !HAS_API_KEY, - }, async () => { - const info = await getAccount({ - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + afterAll(() => { + urlStub.restore(); }); - assertArrayIncludes(Object.keys(info).sort(), [ - "account_email", - "account_id", - "account_rate_limit_per_hour", - "api_key", - "extra_credits", - "last_hour_searches", - "plan_id", - "plan_monthly_price", - "plan_name", - "plan_searches_left", - "searches_per_month", - "this_hour_searches", - "this_month_usage", - "total_searches_left", - ]); - }); - - it("callback", { - ignore: !HAS_API_KEY, - }, async () => { - const info = await new Promise>>( - (res) => getAccount({ api_key: SERPAPI_TEST_KEY, timeout: 10000 }, res), - ); - assertArrayIncludes(Object.keys(info).sort(), [ - "account_email", - "account_id", - "account_rate_limit_per_hour", - "api_key", - "extra_credits", - "last_hour_searches", - "plan_id", - "plan_monthly_price", - "plan_name", - "plan_searches_left", - "searches_per_month", - "this_hour_searches", - "this_month_usage", - "total_searches_left", - ]); - }); - - it("rely on global config", { - ignore: !HAS_API_KEY, - }, async () => { - config.api_key = SERPAPI_TEST_KEY; - const info = await getAccount(); - assertArrayIncludes(Object.keys(info).sort(), [ - "account_email", - "account_id", - "account_rate_limit_per_hour", - "api_key", - "extra_credits", - "last_hour_searches", - "plan_id", - "plan_monthly_price", - "plan_name", - "plan_searches_left", - "searches_per_month", - "this_hour_searches", - "this_month_usage", - "total_searches_left", - ]); - }); -}); - -describe("getLocations", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with invalid timeout", () => { - assertRejects( - async () => await getLocations({ timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getLocations({ timeout: -10 }), - InvalidTimeoutError, - ); - }); - - it("async/await", async () => { - const locations = await getLocations({ limit: 3 }); - assertInstanceOf(locations, Array); - assertEquals(locations.length, 3); - }); - - it("callback", async () => { - const locations = await new Promise< - Awaited> - >( - (res) => getLocations({ limit: 3 }, res), - ); - assertInstanceOf(locations, Array); - assertEquals(locations.length, 3); - }); - - it("without parameters", async () => { - const locations = await getLocations(); - assertInstanceOf(locations, Array); - }); -}); - -describe("getJson", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with no api_key", () => { - assertRejects( - async () => await getJson({ engine: "google", q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getJson("google", { q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getJson({}), - MissingApiKeyError, - ); - }); - it("with invalid arguments", () => { - assertRejects( - // @ts-ignore testing invalid usage - async () => await getJson("google"), - InvalidArgumentError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getJson(), - InvalidArgumentError, - ); - }); + it("with no api_key", () => { + assertRejects( + async () => await getAccount({ api_key: "" }), + MissingApiKeyError, + ); + assertRejects(async () => await getAccount({}), MissingApiKeyError); + assertRejects(async () => await getAccount(), MissingApiKeyError); + }); - it("with invalid timeout", () => { - config.api_key = "test_api_key"; - assertRejects( - async () => await getJson({ engine: "google", q: "Paris", timeout: 0 }), - InvalidTimeoutError, + it( + "with invalid timeout", + { + ignore: !HAS_API_KEY, + }, + () => { + config.api_key = SERPAPI_TEST_KEY; + assertRejects( + async () => await getAccount({ timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getAccount({ timeout: -10 }), + InvalidTimeoutError, + ); + }, ); - assertRejects( - async () => await getJson({ engine: "google", q: "Paris", timeout: -10 }), - InvalidTimeoutError, + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const info = await getAccount({ + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertArrayIncludes(Object.keys(info).sort(), [ + "account_email", + "account_id", + "account_rate_limit_per_hour", + "api_key", + "extra_credits", + "last_hour_searches", + "plan_id", + "plan_monthly_price", + "plan_name", + "plan_searches_left", + "searches_per_month", + "this_hour_searches", + "this_month_usage", + "total_searches_left", + ]); + }, ); - assertRejects( - async () => await getJson("google", { q: "Paris", timeout: 0 }), - InvalidTimeoutError, + + it( + "callback", + { + ignore: !HAS_API_KEY, + }, + async () => { + const info = await new Promise>>( + (res) => + getAccount({ api_key: SERPAPI_TEST_KEY, timeout: 10000 }, res), + ); + assertArrayIncludes(Object.keys(info).sort(), [ + "account_email", + "account_id", + "account_rate_limit_per_hour", + "api_key", + "extra_credits", + "last_hour_searches", + "plan_id", + "plan_monthly_price", + "plan_name", + "plan_searches_left", + "searches_per_month", + "this_hour_searches", + "this_month_usage", + "total_searches_left", + ]); + }, ); - assertRejects( - async () => await getJson("google", { q: "Paris", timeout: -10 }), - InvalidTimeoutError, + + it( + "rely on global config", + { + ignore: !HAS_API_KEY, + }, + async () => { + config.api_key = SERPAPI_TEST_KEY; + const info = await getAccount(); + assertArrayIncludes(Object.keys(info).sort(), [ + "account_email", + "account_id", + "account_rate_limit_per_hour", + "api_key", + "extra_credits", + "last_hour_searches", + "plan_id", + "plan_monthly_price", + "plan_name", + "plan_searches_left", + "searches_per_month", + "this_hour_searches", + "this_month_usage", + "total_searches_left", + ]); + }, ); - }); + }, +); + +describe( + "getLocations", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - it("async/await", { - ignore: !HAS_API_KEY, - }, async () => { - const json = await getJson({ - engine: "google", - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + afterAll(() => { + urlStub.restore(); }); - assertEquals(json.search_metadata["status"], "Success"); - assertExists(json.organic_results); - - // old API - const json2 = await getJson("google", { - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + + it("with invalid timeout", () => { + assertRejects( + async () => await getLocations({ timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getLocations({ timeout: -10 }), + InvalidTimeoutError, + ); }); - assertEquals(json2.search_metadata.id, json.search_metadata.id); - }); - it("callback", { - ignore: !HAS_API_KEY, - }, async () => { - const json = await new Promise((done) => { - getJson({ - engine: "google", - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + it("async/await", async () => { + const locations = await getLocations({ limit: 3 }); + assertInstanceOf(locations, Array); + assertEquals(locations.length, 3); }); - assertEquals(json.search_metadata["status"], "Success"); - assertExists(json.organic_results); - // old API - const json2 = await new Promise((done) => { - getJson("google", { - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + it("callback", async () => { + const locations = await new Promise< + Awaited> + >((res) => getLocations({ limit: 3 }, res)); + assertInstanceOf(locations, Array); + assertEquals(locations.length, 3); + }); + + it("without parameters", async () => { + const locations = await getLocations(); + assertInstanceOf(locations, Array); + }); + }, +); + +describe( + "getJson", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); + + afterEach(() => { + config.api_key = null; + }); + + afterAll(() => { + urlStub.restore(); + }); + + it("with no api_key", () => { + assertRejects( + async () => await getJson({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson({}), + MissingApiKeyError, + ); + }); + + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson("google"), + InvalidArgumentError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson(), + InvalidArgumentError, + ); + }); + + it("with invalid timeout", () => { + config.api_key = "test_api_key"; + assertRejects( + async () => await getJson({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => + await getJson({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); }); - assertEquals(json2.search_metadata.id, json.search_metadata.id); - }); - - it("rely on global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_api_key"; - try { - await getJson({ + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const json = await getJson({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + + // old API + const json2 = await getJson("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(json2.search_metadata.id, json.search_metadata.id); + }, + ); + + it( + "callback", + { + ignore: !HAS_API_KEY, + }, + async () => { + const json = await new Promise((done) => { + getJson( + { + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + + // old API + const json2 = await new Promise((done) => { + getJson( + "google", + { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(json2.search_metadata.id, json.search_metadata.id); + }, + ); + + it("rely on global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_api_key"; + try { + await getJson({ + engine: "google", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_api_key", engine: "google", + output: "json", q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_api_key", - engine: "google", - output: "json", - q: "Paris", }); - }); - it("api_key param overrides global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_initial_api_key"; - try { - await getJson({ - engine: "google", + it("api_key param overrides global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_initial_api_key"; + try { + await getJson({ + engine: "google", + api_key: "test_override_api_key", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { api_key: "test_override_api_key", + engine: "google", + output: "json", q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_override_api_key", - engine: "google", - output: "json", - q: "Paris", }); - }); -}); - -describe("getHtml", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with no api_key", () => { - assertRejects( - async () => await getHtml({ engine: "google", q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getHtml("google", { q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getHtml({}), - MissingApiKeyError, - ); - }); + }, +); + +describe( + "getHtml", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - it("with invalid arguments", () => { - assertRejects( - // @ts-ignore testing invalid usage - async () => await getHtml("google"), - InvalidArgumentError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getHtml(), - InvalidArgumentError, - ); - }); + afterEach(() => { + config.api_key = null; + }); - it("with invalid timeout", { - ignore: !HAS_API_KEY, - }, () => { - config.api_key = SERPAPI_TEST_KEY; - assertRejects( - async () => await getHtml({ engine: "google", q: "Paris", timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getHtml({ engine: "google", q: "Paris", timeout: -10 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getHtml("google", { q: "Paris", timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getHtml("google", { q: "Paris", timeout: -10 }), - InvalidTimeoutError, - ); - }); + afterAll(() => { + urlStub.restore(); + }); - it("async/await", { - ignore: !HAS_API_KEY, - }, async () => { - const html = await getHtml({ - engine: "google", - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + it("with no api_key", () => { + assertRejects( + async () => await getHtml({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml({}), + MissingApiKeyError, + ); }); - assertEquals(html.includes("Paris"), true); - // old API - const html2 = await getHtml("google", { - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml("google"), + InvalidArgumentError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml(), + InvalidArgumentError, + ); }); - assertEquals(html2, html); - }); - it("callback", { - ignore: !HAS_API_KEY, - }, async () => { - const html = await new Promise((done) => { - getHtml({ + it( + "with invalid timeout", + { + ignore: !HAS_API_KEY, + }, + () => { + config.api_key = SERPAPI_TEST_KEY; + assertRejects( + async () => + await getHtml({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => + await getHtml({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + }, + ); + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const html = await getHtml({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(html.includes("Paris"), true); + + // old API + const html2 = await getHtml("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(html2, html); + }, + ); + + it( + "callback", + { + ignore: !HAS_API_KEY, + }, + async () => { + const html = await new Promise((done) => { + getHtml( + { + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(html.includes("Paris"), true); + + // old API + const html2 = await new Promise((done) => { + getHtml( + "google", + { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(html2, html); + }, + ); + + it("rely on global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_api_key"; + try { + await getHtml({ + engine: "google", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_api_key", engine: "google", + output: "html", q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + }); }); - assertEquals(html.includes("Paris"), true); - // old API - const html2 = await new Promise((done) => { - getHtml("google", { + it("api_key param overrides global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_initial_api_key"; + try { + await getHtml({ + engine: "google", + api_key: "test_override_api_key", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_override_api_key", + engine: "google", + output: "html", q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + }); }); - assertEquals(html2, html); - }); - - it("rely on global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_api_key"; - try { - await getHtml({ + }, +); + +describe( + "getJsonBySearchId", + { + sanitizeOps: false, + sanitizeResources: false, + ignore: !HAS_API_KEY, + }, + () => { + let id: string; + + beforeAll(async () => { + const response = await getJson({ engine: "google", + api_key: SERPAPI_TEST_KEY, q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_api_key", - engine: "google", - output: "html", - q: "Paris", + let status; + ({ id, status } = response["search_metadata"]); + assert(id, "Missing search id"); + assertEquals(status, "Success"); + }); + + it("getJsonBySearchId (async/await)", async () => { + const json = await getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }); + assertArrayIncludes(Object.keys(json).sort(), ["organic_results"]); }); - }); - it("api_key param overrides global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_initial_api_key"; - try { - await getHtml({ + it("getJsonBySearchId (callback)", async () => { + const json = await new Promise< + Awaited> + >((res) => getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); + assertArrayIncludes(Object.keys(json).sort(), ["organic_results"]); + }); + }, +); + +describe( + "getHtmlBySearchId", + { + sanitizeOps: false, + sanitizeResources: false, + ignore: !HAS_API_KEY, + }, + () => { + let id: string; + + beforeAll(async () => { + const response = await getJson({ engine: "google", - api_key: "test_override_api_key", + api_key: SERPAPI_TEST_KEY, q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_override_api_key", - engine: "google", - output: "html", - q: "Paris", + let status; + ({ id, status } = response["search_metadata"]); + assert(id, "Missing search id"); + assertEquals(status, "Success"); }); - }); -}); - -describe("getJsonBySearchId", { - sanitizeOps: false, - sanitizeResources: false, - ignore: !HAS_API_KEY, -}, () => { - let id: string; - - beforeAll(async () => { - const response = await getJson({ - engine: "google", - api_key: SERPAPI_TEST_KEY, - q: "Paris", + + it("getHtmlBySearchId (async/await)", async () => { + const html = await getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }); + assertStringIncludes(html, ""); + assertStringIncludes(html, ""); }); - let status; - ({ id, status } = response["search_metadata"]); - assert(id, "Missing search id"); - assertEquals(status, "Success"); - }); - - it("getJsonBySearchId (async/await)", async () => { - const json = await getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }); - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - ]); - }); - - it("getJsonBySearchId (callback)", async () => { - const json = await new Promise< - Awaited> - >((res) => getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - ]); - }); -}); - -describe("getHtmlBySearchId", { - sanitizeOps: false, - sanitizeResources: false, - ignore: !HAS_API_KEY, -}, () => { - let id: string; - - beforeAll(async () => { - const response = await getJson({ - engine: "google", - api_key: SERPAPI_TEST_KEY, - q: "Paris", + + it("getHtmlBySearchId (callback)", async () => { + const html = await new Promise< + Awaited> + >((res) => getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); + assertStringIncludes(html, ""); + assertStringIncludes(html, ""); }); - let status; - ({ id, status } = response["search_metadata"]); - assert(id, "Missing search id"); - assertEquals(status, "Success"); - }); - - it("getHtmlBySearchId (async/await)", async () => { - const html = await getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }); - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); - - it("getHtmlBySearchId (callback)", async () => { - const html = await new Promise< - Awaited> - >((res) => getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); -}); + }, +); diff --git a/tests/utils_test.ts b/tests/utils_test.ts index f66b37e..4d7e32d 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -11,13 +11,24 @@ import { assertInstanceOf, assertMatch, } from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals, buildUrl, execute, getSource } from "../src/utils.ts"; +import { + _internals, + buildRequestOptions, + execute, + getSource, +} from "../src/utils.ts"; import { RequestTimeoutError } from "../src/errors.ts"; loadSync({ export: true }); -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; +const BASE_OPTIONS = Deno.env.get("ENV_TYPE") === "local" + ? { + hostname: "localhost", + port: 3000, + } + : { + hostname: "serpapi.com", + port: 443, + }; describe("getSource", () => { it("use runtime version", async () => { @@ -28,11 +39,11 @@ describe("getSource", () => { }); }); -describe("buildUrl", () => { +describe("buildRequestOptions", () => { let urlStub: Stub; beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); }); afterAll(() => { @@ -40,55 +51,81 @@ describe("buildUrl", () => { }); it("with blank path and empty parameters", async () => { - assertEquals(await buildUrl("", {}), `${BASE_URL}?`); + assertEquals(await buildRequestOptions("", {}), { + ...BASE_OPTIONS, + method: "GET", + path: "?", + }); }); it("with path and empty parameters", async () => { - assertEquals(await buildUrl("/", {}), `${BASE_URL}/?`); + assertEquals(await buildRequestOptions("/", {}), { + ...BASE_OPTIONS, + method: "GET", + path: "/?", + }); }); it("with path and parameters", async () => { assertEquals( - await buildUrl("/search", { q: "coffee", gl: "us" }), - `${BASE_URL}/search?q=coffee&gl=us`, + await buildRequestOptions("/search", { q: "coffee", gl: "us" }), + { + ...BASE_OPTIONS, + method: "GET", + path: "/search?q=coffee&gl=us", + }, ); }); it("with source", async () => { - const url = await buildUrl("/search", { source: await getSource() }); + const options = await buildRequestOptions("/search", { + source: await getSource(), + }); assertMatch( - url, + options.path as string, /source=(nodejs|deno)%40\d+\.\d+\.\d+%2Cserpapi%40\d+\.\d+\.\d+$/, ); }); it("with undefined parameters", async () => { assertEquals( - await buildUrl("/search", { q: "coffee", gl: undefined, hl: null }), - `${BASE_URL}/search?q=coffee&hl=`, + await buildRequestOptions("/search", { + q: "coffee", + gl: undefined, + hl: null, + }), + { + ...BASE_OPTIONS, + method: "GET", + path: "/search?q=coffee&hl=", + }, ); }); }); -describe("execute", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; +describe( + "execute", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - afterAll(() => { - urlStub.restore(); - }); + afterAll(() => { + urlStub.restore(); + }); - it("with short timeout", async () => { - try { - await execute("/search", { q: "coffee", gl: "us" }, 1); - } catch (e) { - assertInstanceOf(e, RequestTimeoutError); - } - }); -}); + it("with short timeout", async () => { + try { + await execute("/search", { q: "coffee", gl: "us" }, 1); + } catch (e) { + assertInstanceOf(e, RequestTimeoutError); + } + }); + }, +); From 291b207e112efdf13b6322a83189bd9c008860f2 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 7 Apr 2025 09:21:40 +0800 Subject: [PATCH 09/12] Add tests for requestOptions --- src/utils.ts | 8 +++- tests/utils_test.ts | 95 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index ff3f88f..c90d608 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -49,7 +49,11 @@ export function buildRequestOptions( ): http.RequestOptions { const clonedParams = { ...parameters }; for (const k in clonedParams) { - if (clonedParams[k] === undefined) { + if ( + k === "requestOptions" || + k === "timeout" || + clonedParams[k] === undefined + ) { delete clonedParams[k]; } } @@ -61,8 +65,8 @@ export function buildRequestOptions( return { ...base, - ...(parameters.requestOptions as http.RequestOptions), ...config.requestOptions, + ...(parameters.requestOptions as http.RequestOptions), }; } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index 4d7e32d..eafe544 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -18,6 +18,9 @@ import { getSource, } from "../src/utils.ts"; import { RequestTimeoutError } from "../src/errors.ts"; +import { Config, config } from "../src/config.ts"; +import http from "node:http"; +import qs from "node:querystring"; loadSync({ export: true }); const BASE_OPTIONS = Deno.env.get("ENV_TYPE") === "local" @@ -101,6 +104,98 @@ describe("buildRequestOptions", () => { }, ); }); + + describe("with requestOptions", () => { + let originalConfig: Config; + + beforeAll(() => { + originalConfig = { ...config }; + }); + + afterAll(() => { + Object.assign(config, originalConfig); + }); + + it("uses default options when no custom options provided", async () => { + const options = await buildRequestOptions("/search", { q: "coffee" }); + assertEquals(options.method, "GET"); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("uses custom request options from parameters", async () => { + const customOptions: http.RequestOptions = { + headers: { + "User-Agent": "Custom User Agent", + "X-Custom-Header": "param-value", + }, + timeout: 5000, + }; + + const params = { + q: "coffee", + requestOptions: customOptions, + } as unknown as qs.ParsedUrlQueryInput; + + const options = await buildRequestOptions("/search", params); + + assertEquals(options.headers?.["User-Agent"], "Custom User Agent"); + assertEquals(options.headers?.["X-Custom-Header"], "param-value"); + assertEquals(options.timeout, 5000); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("uses request options from config when no options in parameters", async () => { + const configOptions: http.RequestOptions = { + headers: { + "User-Agent": "Config User Agent", + "X-Custom-Header": "config-value", + }, + timeout: 5000, + }; + + config.requestOptions = configOptions; + + const options = await buildRequestOptions("/search", { q: "coffee" }); + + assertEquals(options.headers?.["User-Agent"], "Config User Agent"); + assertEquals(options.headers?.["X-Custom-Header"], "config-value"); + assertEquals(options.timeout, 5000); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("parameters requestOptions merges over config options", async () => { + const configOptions: http.RequestOptions = { + headers: { + "User-Agent": "Config User Agent", + "X-Custom-Header": "config-value", + }, + timeout: 5000, + }; + + const paramOptions: http.RequestOptions = { + headers: { + "User-Agent": "Parameter User Agent", + "X-Custom-Header": "param-value", + }, + hostname: "localhost", + }; + + config.requestOptions = configOptions; + + const params = { + q: "coffee", + requestOptions: paramOptions, + } as unknown as qs.ParsedUrlQueryInput; + + const options = await buildRequestOptions("/search", params); + + assertEquals(options.headers?.["User-Agent"], "Parameter User Agent"); + assertEquals(options.headers?.["X-Custom-Header"], "param-value"); + assertEquals(options.hostname, "localhost"); + assertEquals(options.timeout, 5000); + assertEquals(options.path, "/search?q=coffee"); + }); + }); }); describe( From faca4778246f9c1adb7c2e9a431e79d1194641a7 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 7 Apr 2025 09:46:04 +0800 Subject: [PATCH 10/12] Update README --- README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/README.md b/README.md index 6838391..211f26c 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,39 @@ await getJson({ engine: "google", q: "coffee" }); // uses the API key defined in await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used ``` +### Using a Proxy + +You can use a proxy by passing `requestOptions` with an `HttpsProxyAgent` +instance. This can be done either globally through the config object or +per-request in the parameters. + +First, install the required package: + +```bash +npm install https-proxy-agent +``` + +Then use it in your code: + +```js +import { config, getJson } from "serpapi"; +import { HttpsProxyAgent } from "https-proxy-agent"; + +// Global configuration +config.requestOptions = { + agent: new HttpsProxyAgent("http://proxy-server:port"), +}; + +// Or per-request configuration +await getJson({ + engine: "google", + q: "coffee", + requestOptions: { + agent: new HttpsProxyAgent("http://proxy-server:port"), + }, +}); +``` + ## Pagination Built-in pagination is not supported. Please refer to our pagination examples From 6b45021973fd552fcc20b5c1c4fe1123e55adbe4 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 7 Apr 2025 10:06:05 +0800 Subject: [PATCH 11/12] Disallow modifying basic request options --- src/utils.ts | 4 ++-- tests/utils_test.ts | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index c90d608..1d9c8ea 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -57,16 +57,16 @@ export function buildRequestOptions( delete clonedParams[k]; } } - const base = { + const basicOptions = { ..._internals.getHostnameAndPort(), path: `${path}?${qs.stringify(clonedParams)}`, method: "GET", }; return { - ...base, ...config.requestOptions, ...(parameters.requestOptions as http.RequestOptions), + ...basicOptions, }; } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index eafe544..cc23415 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -177,7 +177,7 @@ describe("buildRequestOptions", () => { "User-Agent": "Parameter User Agent", "X-Custom-Header": "param-value", }, - hostname: "localhost", + agent: true, }; config.requestOptions = configOptions; @@ -191,10 +191,25 @@ describe("buildRequestOptions", () => { assertEquals(options.headers?.["User-Agent"], "Parameter User Agent"); assertEquals(options.headers?.["X-Custom-Header"], "param-value"); - assertEquals(options.hostname, "localhost"); + assertEquals(options.agent, true); assertEquals(options.timeout, 5000); assertEquals(options.path, "/search?q=coffee"); }); + + it("basic options are not allowed to be changed", async () => { + config.requestOptions = { + hostname: "localhost", + port: 3000, + path: "/test", + method: "POST", + }; + + const options = await buildRequestOptions("/search", { q: "coffee" }); + assertEquals(options.hostname, "serpapi.com"); + assertEquals(options.port, 443); + assertEquals(options.path, "/search?q=coffee"); + assertEquals(options.method, "GET"); + }); }); }); From 9032a1efecae9de30225403a9b146abfd9e2ee7d Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 7 Apr 2025 10:13:25 +0800 Subject: [PATCH 12/12] Remove type annotation for chunk --- src/utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 1d9c8ea..f574584 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -4,7 +4,6 @@ import http from "node:http"; import qs from "node:querystring"; import { RequestTimeoutError } from "./errors.ts"; import { config } from "./config.ts"; -import { Buffer } from "node:buffer"; import process from "node:process"; /** @@ -88,7 +87,7 @@ export function execute( let data = ""; // A chunk of data has been received - resp.on("data", (chunk: Buffer) => { + resp.on("data", (chunk) => { data += chunk; });