diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 83b4b27..392b97f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,15 +5,15 @@ on: [push] jobs: build: name: "Deno tests and build npm files" - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo uses: actions/checkout@v3 - name: Setup Deno - uses: denoland/setup-deno@v1.1.1 + uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Check formatting run: deno fmt --check @@ -27,10 +27,10 @@ jobs: run: deno task test:ci - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: '18.x' # Build files using a fixed node version - registry-url: 'https://registry.npmjs.org' + node-version: "22.x" # Build files using a fixed node version + registry-url: "https://registry.npmjs.org" - name: Build npm files run: deno task npm @@ -39,7 +39,7 @@ jobs: run: zip npm.zip ./npm -r - name: Upload build files for smoke tests - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: npm path: npm.zip @@ -48,10 +48,24 @@ jobs: smoke-tests-commonjs: name: "Smoke tests (CommonJS)" needs: build - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: - node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + node-version: [ + 7.x, + 8.x, + 9.x, + 10.x, + 11.x, + 12.x, + 13.x, + 14.x, + 15.x, + 16.x, + 17.x, + 18.x, + 19.x, + ] include: - command: test - command: test:use-openssl-ca @@ -63,16 +77,16 @@ jobs: steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" - name: Download build files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: npm @@ -90,22 +104,22 @@ jobs: smoke-tests-esm: name: "Smoke tests (ESM)" needs: build - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: matrix: node-version: [14.x, 15.x, 16.x, 17.x, 18.x, 19.x] steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - registry-url: 'https://registry.npmjs.org' + registry-url: "https://registry.npmjs.org" - name: Download build files - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: npm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e35e4e..1297b81 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,19 +5,19 @@ on: branches: - master paths: - - 'version.ts' + - "version.ts" jobs: release: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Deno - uses: denoland/setup-deno@v1.1.1 + uses: denoland/setup-deno@v2 with: - deno-version: v1.x + deno-version: v2.x - name: Check formatting run: deno fmt --check @@ -31,10 +31,10 @@ jobs: run: deno task test:ci - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: - node-version: '18.x' - registry-url: 'https://registry.npmjs.org' + node-version: "22.x" + registry-url: "https://registry.npmjs.org" - name: Build npm files run: deno task npm diff --git a/README.md b/README.md index 6838391..211f26c 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,39 @@ await getJson({ engine: "google", q: "coffee" }); // uses the API key defined in await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used ``` +### Using a Proxy + +You can use a proxy by passing `requestOptions` with an `HttpsProxyAgent` +instance. This can be done either globally through the config object or +per-request in the parameters. + +First, install the required package: + +```bash +npm install https-proxy-agent +``` + +Then use it in your code: + +```js +import { config, getJson } from "serpapi"; +import { HttpsProxyAgent } from "https-proxy-agent"; + +// Global configuration +config.requestOptions = { + agent: new HttpsProxyAgent("http://proxy-server:port"), +}; + +// Or per-request configuration +await getJson({ + engine: "google", + q: "coffee", + requestOptions: { + agent: new HttpsProxyAgent("http://proxy-server:port"), + }, +}); +``` + ## Pagination Built-in pagination is not supported. Please refer to our pagination examples diff --git a/deno.json b/deno.json index 1a6d5f5..03d6ad8 100644 --- a/deno.json +++ b/deno.json @@ -8,9 +8,7 @@ "npm": "deno run -A scripts/build_npm.ts" }, "fmt": { - "files": { - "exclude": ["npm/", "examples/node", "smoke_tests/"] - } + "exclude": ["npm/", "examples/node", "smoke_tests/"] }, "lint": { "files": { diff --git a/examples/node/js_node_14_up/basic_example.js b/examples/node/js_node_14_up/basic_example.js index 41c36cd..9c81fc7 100644 --- a/examples/node/js_node_14_up/basic_example.js +++ b/examples/node/js_node_14_up/basic_example.js @@ -7,6 +7,7 @@ */ import * as Dotenv from "dotenv"; +import process from "process"; import { config, getJson } from "serpapi"; Dotenv.config(); diff --git a/examples/node/js_node_14_up/pagination_example.js b/examples/node/js_node_14_up/pagination_example.js index 07c705a..15619a5 100644 --- a/examples/node/js_node_14_up/pagination_example.js +++ b/examples/node/js_node_14_up/pagination_example.js @@ -7,6 +7,7 @@ */ import * as Dotenv from "dotenv"; +import process from "process"; import { config, getJson } from "serpapi"; Dotenv.config(); diff --git a/examples/node/js_node_7_up/basic_example.js b/examples/node/js_node_7_up/basic_example.js index efe56c2..e03dd61 100644 --- a/examples/node/js_node_7_up/basic_example.js +++ b/examples/node/js_node_7_up/basic_example.js @@ -3,6 +3,7 @@ */ const Dotenv = require("dotenv"); +const process = require("process"); const { config, getJson } = require("serpapi"); Dotenv.config(); diff --git a/examples/node/js_node_7_up/pagination_example.js b/examples/node/js_node_7_up/pagination_example.js index fdaebf8..691b13d 100644 --- a/examples/node/js_node_7_up/pagination_example.js +++ b/examples/node/js_node_7_up/pagination_example.js @@ -6,6 +6,7 @@ const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); const url = require("url"); const qs = require("querystring"); +const process = require("process"); Dotenv.config(); config.api_key = process.env.API_KEY; diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 4b92248..a88a0de 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -1,4 +1,4 @@ -import { build, emptyDir } from "https://deno.land/x/dnt@0.37.0/mod.ts"; +import { build, emptyDir } from "https://deno.land/x/dnt@0.40.0/mod.ts"; import { version } from "../version.ts"; await emptyDir("./npm"); diff --git a/smoke_tests/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js index 46c4ff2..fde3f4e 100644 --- a/smoke_tests/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -3,6 +3,7 @@ */ const Dotenv = require("dotenv"); +const process = require("process"); const { config, getJson, diff --git a/smoke_tests/esm/esm.js b/smoke_tests/esm/esm.js index ad4bee2..080eb85 100644 --- a/smoke_tests/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -7,6 +7,7 @@ */ import Dotenv from "dotenv"; +import process from "process"; import { config, getAccount, diff --git a/src/config.ts b/src/config.ts index 2a17600..386b7ea 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,11 @@ +import http from "node:http"; + export type Config = { api_key: string | null; timeout: number; + requestOptions?: http.RequestOptions; }; + export const config: Config = { api_key: null, timeout: 60000, diff --git a/src/utils.ts b/src/utils.ts index a05fa05..f574584 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,7 +1,10 @@ import { version } from "../version.ts"; import https from "node:https"; +import http from "node:http"; import qs from "node:querystring"; import { RequestTimeoutError } from "./errors.ts"; +import { config } from "./config.ts"; +import process from "node:process"; /** * This `_internals` object is needed to support stubbing/spying of @@ -12,12 +15,15 @@ import { RequestTimeoutError } from "./errors.ts"; */ export const _internals = { execute: execute, - getBaseUrl: getBaseUrl, + getHostnameAndPort: getHostnameAndPort, }; /** Facilitates stubbing in tests, e.g. localhost as the base url */ -function getBaseUrl() { - return "https://serpapi.com"; +function getHostnameAndPort() { + return { + hostname: "serpapi.com", + port: 443, + }; } export function getSource() { @@ -27,9 +33,7 @@ export function getSource() { if (denoVersion) { return `deno@${denoVersion},${moduleSource}`; } - // @ts-ignore: scope of nodejs } else if (typeof process == "object") { - // @ts-ignore: scope of nodejs const nodeVersion = process.versions?.node; if (nodeVersion) { return `nodejs@${nodeVersion},${moduleSource}`; @@ -38,17 +42,31 @@ export function getSource() { return `nodejs,${moduleSource}`; } -export function buildUrl( +export function buildRequestOptions( path: string, parameters: qs.ParsedUrlQueryInput, -): string { +): http.RequestOptions { const clonedParams = { ...parameters }; for (const k in clonedParams) { - if (clonedParams[k] === undefined) { + if ( + k === "requestOptions" || + k === "timeout" || + clonedParams[k] === undefined + ) { delete clonedParams[k]; } } - return `${_internals.getBaseUrl()}${path}?${qs.stringify(clonedParams)}`; + const basicOptions = { + ..._internals.getHostnameAndPort(), + path: `${path}?${qs.stringify(clonedParams)}`, + method: "GET", + }; + + return { + ...config.requestOptions, + ...(parameters.requestOptions as http.RequestOptions), + ...basicOptions, + }; } export function execute( @@ -56,22 +74,24 @@ export function execute( parameters: qs.ParsedUrlQueryInput, timeout: number, ): Promise { - const url = buildUrl(path, { + const options = buildRequestOptions(path, { ...parameters, source: getSource(), }); + return new Promise((resolve, reject) => { let timer: number; - const req = https.get(url, (resp) => { + + const handleResponse = (resp: http.IncomingMessage) => { resp.setEncoding("utf8"); let data = ""; - // A chunk of data has been recieved. + // A chunk of data has been received resp.on("data", (chunk) => { data += chunk; }); - // The whole response has been received. Print out the result. + // The whole response has been received resp.on("end", () => { try { if (resp.statusCode == 200) { @@ -85,10 +105,14 @@ export function execute( if (timer) clearTimeout(timer); } }); - }).on("error", (err) => { + }; + + const handleError = (err: Error) => { reject(err); if (timer) clearTimeout(timer); - }); + }; + + const req = https.get(options, handleResponse).on("error", handleError); if (timeout > 0) { timer = setTimeout(() => { diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index ea58049..e22143b 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -40,565 +40,637 @@ import { loadSync({ export: true }); const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("getAccount", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with no api_key", () => { - assertRejects( - async () => await getAccount({ api_key: "" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getAccount({}), - MissingApiKeyError, - ); - assertRejects( - async () => await getAccount(), - MissingApiKeyError, - ); - }); +const BASE_OPTIONS = Deno.env.get("ENV_TYPE") === "local" + ? { + hostname: "localhost", + port: 3000, + } + : { + hostname: "serpapi.com", + port: 443, + }; + +describe( + "getAccount", + { + sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - it("with invalid timeout", { - ignore: !HAS_API_KEY, - }, () => { - config.api_key = SERPAPI_TEST_KEY; - assertRejects( - async () => await getAccount({ timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getAccount({ timeout: -10 }), - InvalidTimeoutError, - ); - }); + afterEach(() => { + config.api_key = null; + }); - it("async/await", { - ignore: !HAS_API_KEY, - }, async () => { - const info = await getAccount({ - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + afterAll(() => { + urlStub.restore(); }); - assertArrayIncludes(Object.keys(info).sort(), [ - "account_email", - "account_id", - "account_rate_limit_per_hour", - "api_key", - "extra_credits", - "last_hour_searches", - "plan_id", - "plan_monthly_price", - "plan_name", - "plan_searches_left", - "searches_per_month", - "this_hour_searches", - "this_month_usage", - "total_searches_left", - ]); - }); - - it("callback", { - ignore: !HAS_API_KEY, - }, async () => { - const info = await new Promise>>( - (res) => getAccount({ api_key: SERPAPI_TEST_KEY, timeout: 10000 }, res), - ); - assertArrayIncludes(Object.keys(info).sort(), [ - "account_email", - "account_id", - "account_rate_limit_per_hour", - "api_key", - "extra_credits", - "last_hour_searches", - "plan_id", - "plan_monthly_price", - "plan_name", - "plan_searches_left", - "searches_per_month", - "this_hour_searches", - "this_month_usage", - "total_searches_left", - ]); - }); - - it("rely on global config", { - ignore: !HAS_API_KEY, - }, async () => { - config.api_key = SERPAPI_TEST_KEY; - const info = await getAccount(); - assertArrayIncludes(Object.keys(info).sort(), [ - "account_email", - "account_id", - "account_rate_limit_per_hour", - "api_key", - "extra_credits", - "last_hour_searches", - "plan_id", - "plan_monthly_price", - "plan_name", - "plan_searches_left", - "searches_per_month", - "this_hour_searches", - "this_month_usage", - "total_searches_left", - ]); - }); -}); - -describe("getLocations", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with invalid timeout", () => { - assertRejects( - async () => await getLocations({ timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getLocations({ timeout: -10 }), - InvalidTimeoutError, - ); - }); - - it("async/await", async () => { - const locations = await getLocations({ limit: 3 }); - assertInstanceOf(locations, Array); - assertEquals(locations.length, 3); - }); - - it("callback", async () => { - const locations = await new Promise< - Awaited> - >( - (res) => getLocations({ limit: 3 }, res), - ); - assertInstanceOf(locations, Array); - assertEquals(locations.length, 3); - }); - - it("without parameters", async () => { - const locations = await getLocations(); - assertInstanceOf(locations, Array); - }); -}); - -describe("getJson", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with no api_key", () => { - assertRejects( - async () => await getJson({ engine: "google", q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getJson("google", { q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getJson({}), - MissingApiKeyError, - ); - }); - it("with invalid arguments", () => { - assertRejects( - // @ts-ignore testing invalid usage - async () => await getJson("google"), - InvalidArgumentError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getJson(), - InvalidArgumentError, - ); - }); + it("with no api_key", () => { + assertRejects( + async () => await getAccount({ api_key: "" }), + MissingApiKeyError, + ); + assertRejects(async () => await getAccount({}), MissingApiKeyError); + assertRejects(async () => await getAccount(), MissingApiKeyError); + }); - it("with invalid timeout", () => { - config.api_key = "test_api_key"; - assertRejects( - async () => await getJson({ engine: "google", q: "Paris", timeout: 0 }), - InvalidTimeoutError, + it( + "with invalid timeout", + { + ignore: !HAS_API_KEY, + }, + () => { + config.api_key = SERPAPI_TEST_KEY; + assertRejects( + async () => await getAccount({ timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getAccount({ timeout: -10 }), + InvalidTimeoutError, + ); + }, ); - assertRejects( - async () => await getJson({ engine: "google", q: "Paris", timeout: -10 }), - InvalidTimeoutError, + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const info = await getAccount({ + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertArrayIncludes(Object.keys(info).sort(), [ + "account_email", + "account_id", + "account_rate_limit_per_hour", + "api_key", + "extra_credits", + "last_hour_searches", + "plan_id", + "plan_monthly_price", + "plan_name", + "plan_searches_left", + "searches_per_month", + "this_hour_searches", + "this_month_usage", + "total_searches_left", + ]); + }, ); - assertRejects( - async () => await getJson("google", { q: "Paris", timeout: 0 }), - InvalidTimeoutError, + + it( + "callback", + { + ignore: !HAS_API_KEY, + }, + async () => { + const info = await new Promise>>( + (res) => + getAccount({ api_key: SERPAPI_TEST_KEY, timeout: 10000 }, res), + ); + assertArrayIncludes(Object.keys(info).sort(), [ + "account_email", + "account_id", + "account_rate_limit_per_hour", + "api_key", + "extra_credits", + "last_hour_searches", + "plan_id", + "plan_monthly_price", + "plan_name", + "plan_searches_left", + "searches_per_month", + "this_hour_searches", + "this_month_usage", + "total_searches_left", + ]); + }, ); - assertRejects( - async () => await getJson("google", { q: "Paris", timeout: -10 }), - InvalidTimeoutError, + + it( + "rely on global config", + { + ignore: !HAS_API_KEY, + }, + async () => { + config.api_key = SERPAPI_TEST_KEY; + const info = await getAccount(); + assertArrayIncludes(Object.keys(info).sort(), [ + "account_email", + "account_id", + "account_rate_limit_per_hour", + "api_key", + "extra_credits", + "last_hour_searches", + "plan_id", + "plan_monthly_price", + "plan_name", + "plan_searches_left", + "searches_per_month", + "this_hour_searches", + "this_month_usage", + "total_searches_left", + ]); + }, ); - }); + }, +); + +describe( + "getLocations", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - it("async/await", { - ignore: !HAS_API_KEY, - }, async () => { - const json = await getJson({ - engine: "google", - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + afterAll(() => { + urlStub.restore(); }); - assertEquals(json.search_metadata["status"], "Success"); - assertExists(json.organic_results); - - // old API - const json2 = await getJson("google", { - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + + it("with invalid timeout", () => { + assertRejects( + async () => await getLocations({ timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getLocations({ timeout: -10 }), + InvalidTimeoutError, + ); }); - assertEquals(json2.search_metadata.id, json.search_metadata.id); - }); - it("callback", { - ignore: !HAS_API_KEY, - }, async () => { - const json = await new Promise((done) => { - getJson({ - engine: "google", - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + it("async/await", async () => { + const locations = await getLocations({ limit: 3 }); + assertInstanceOf(locations, Array); + assertEquals(locations.length, 3); }); - assertEquals(json.search_metadata["status"], "Success"); - assertExists(json.organic_results); - // old API - const json2 = await new Promise((done) => { - getJson("google", { - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + it("callback", async () => { + const locations = await new Promise< + Awaited> + >((res) => getLocations({ limit: 3 }, res)); + assertInstanceOf(locations, Array); + assertEquals(locations.length, 3); + }); + + it("without parameters", async () => { + const locations = await getLocations(); + assertInstanceOf(locations, Array); + }); + }, +); + +describe( + "getJson", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); + + afterEach(() => { + config.api_key = null; + }); + + afterAll(() => { + urlStub.restore(); + }); + + it("with no api_key", () => { + assertRejects( + async () => await getJson({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson({}), + MissingApiKeyError, + ); + }); + + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson("google"), + InvalidArgumentError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson(), + InvalidArgumentError, + ); + }); + + it("with invalid timeout", () => { + config.api_key = "test_api_key"; + assertRejects( + async () => await getJson({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => + await getJson({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); }); - assertEquals(json2.search_metadata.id, json.search_metadata.id); - }); - - it("rely on global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_api_key"; - try { - await getJson({ + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const json = await getJson({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + + // old API + const json2 = await getJson("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(json2.search_metadata.id, json.search_metadata.id); + }, + ); + + it( + "callback", + { + ignore: !HAS_API_KEY, + }, + async () => { + const json = await new Promise((done) => { + getJson( + { + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + + // old API + const json2 = await new Promise((done) => { + getJson( + "google", + { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(json2.search_metadata.id, json.search_metadata.id); + }, + ); + + it("rely on global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_api_key"; + try { + await getJson({ + engine: "google", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_api_key", engine: "google", + output: "json", q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_api_key", - engine: "google", - output: "json", - q: "Paris", }); - }); - it("api_key param overrides global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_initial_api_key"; - try { - await getJson({ - engine: "google", + it("api_key param overrides global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_initial_api_key"; + try { + await getJson({ + engine: "google", + api_key: "test_override_api_key", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { api_key: "test_override_api_key", + engine: "google", + output: "json", q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_override_api_key", - engine: "google", - output: "json", - q: "Paris", }); - }); -}); - -describe("getHtml", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("with no api_key", () => { - assertRejects( - async () => await getHtml({ engine: "google", q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getHtml("google", { q: "Paris" }), - MissingApiKeyError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getHtml({}), - MissingApiKeyError, - ); - }); + }, +); + +describe( + "getHtml", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); - it("with invalid arguments", () => { - assertRejects( - // @ts-ignore testing invalid usage - async () => await getHtml("google"), - InvalidArgumentError, - ); - assertRejects( - // @ts-ignore testing invalid usage - async () => await getHtml(), - InvalidArgumentError, - ); - }); + afterEach(() => { + config.api_key = null; + }); - it("with invalid timeout", { - ignore: !HAS_API_KEY, - }, () => { - config.api_key = SERPAPI_TEST_KEY; - assertRejects( - async () => await getHtml({ engine: "google", q: "Paris", timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getHtml({ engine: "google", q: "Paris", timeout: -10 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getHtml("google", { q: "Paris", timeout: 0 }), - InvalidTimeoutError, - ); - assertRejects( - async () => await getHtml("google", { q: "Paris", timeout: -10 }), - InvalidTimeoutError, - ); - }); + afterAll(() => { + urlStub.restore(); + }); - it("async/await", { - ignore: !HAS_API_KEY, - }, async () => { - const html = await getHtml({ - engine: "google", - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + it("with no api_key", () => { + assertRejects( + async () => await getHtml({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml({}), + MissingApiKeyError, + ); }); - assertEquals(html.includes("Paris"), true); - // old API - const html2 = await getHtml("google", { - q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml("google"), + InvalidArgumentError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml(), + InvalidArgumentError, + ); }); - assertEquals(html2, html); - }); - it("callback", { - ignore: !HAS_API_KEY, - }, async () => { - const html = await new Promise((done) => { - getHtml({ + it( + "with invalid timeout", + { + ignore: !HAS_API_KEY, + }, + () => { + config.api_key = SERPAPI_TEST_KEY; + assertRejects( + async () => + await getHtml({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => + await getHtml({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + }, + ); + + it( + "async/await", + { + ignore: !HAS_API_KEY, + }, + async () => { + const html = await getHtml({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(html.includes("Paris"), true); + + // old API + const html2 = await getHtml("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(html2, html); + }, + ); + + it( + "callback", + { + ignore: !HAS_API_KEY, + }, + async () => { + const html = await new Promise((done) => { + getHtml( + { + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(html.includes("Paris"), true); + + // old API + const html2 = await new Promise((done) => { + getHtml( + "google", + { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, + done, + ); + }); + assertEquals(html2, html); + }, + ); + + it("rely on global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_api_key"; + try { + await getHtml({ + engine: "google", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_api_key", engine: "google", + output: "html", q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + }); }); - assertEquals(html.includes("Paris"), true); - // old API - const html2 = await new Promise((done) => { - getHtml("google", { + it("api_key param overrides global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_initial_api_key"; + try { + await getHtml({ + engine: "google", + api_key: "test_override_api_key", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_override_api_key", + engine: "google", + output: "html", q: "Paris", - api_key: SERPAPI_TEST_KEY, - timeout: 10000, - }, done); + }); }); - assertEquals(html2, html); - }); - - it("rely on global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_api_key"; - try { - await getHtml({ + }, +); + +describe( + "getJsonBySearchId", + { + sanitizeOps: false, + sanitizeResources: false, + ignore: !HAS_API_KEY, + }, + () => { + let id: string; + + beforeAll(async () => { + const response = await getJson({ engine: "google", + api_key: SERPAPI_TEST_KEY, q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_api_key", - engine: "google", - output: "html", - q: "Paris", + let status; + ({ id, status } = response["search_metadata"]); + assert(id, "Missing search id"); + assertEquals(status, "Success"); + }); + + it("getJsonBySearchId (async/await)", async () => { + const json = await getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }); + assertArrayIncludes(Object.keys(json).sort(), ["organic_results"]); }); - }); - it("api_key param overrides global config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_initial_api_key"; - try { - await getHtml({ + it("getJsonBySearchId (callback)", async () => { + const json = await new Promise< + Awaited> + >((res) => getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); + assertArrayIncludes(Object.keys(json).sort(), ["organic_results"]); + }); + }, +); + +describe( + "getHtmlBySearchId", + { + sanitizeOps: false, + sanitizeResources: false, + ignore: !HAS_API_KEY, + }, + () => { + let id: string; + + beforeAll(async () => { + const response = await getJson({ engine: "google", - api_key: "test_override_api_key", + api_key: SERPAPI_TEST_KEY, q: "Paris", }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_override_api_key", - engine: "google", - output: "html", - q: "Paris", + let status; + ({ id, status } = response["search_metadata"]); + assert(id, "Missing search id"); + assertEquals(status, "Success"); }); - }); -}); - -describe("getJsonBySearchId", { - sanitizeOps: false, - sanitizeResources: false, - ignore: !HAS_API_KEY, -}, () => { - let id: string; - - beforeAll(async () => { - const response = await getJson({ - engine: "google", - api_key: SERPAPI_TEST_KEY, - q: "Paris", + + it("getHtmlBySearchId (async/await)", async () => { + const html = await getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }); + assertStringIncludes(html, ""); + assertStringIncludes(html, ""); }); - let status; - ({ id, status } = response["search_metadata"]); - assert(id, "Missing search id"); - assertEquals(status, "Success"); - }); - - it("getJsonBySearchId (async/await)", async () => { - const json = await getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }); - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - ]); - }); - - it("getJsonBySearchId (callback)", async () => { - const json = await new Promise< - Awaited> - >((res) => getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - ]); - }); -}); - -describe("getHtmlBySearchId", { - sanitizeOps: false, - sanitizeResources: false, - ignore: !HAS_API_KEY, -}, () => { - let id: string; - - beforeAll(async () => { - const response = await getJson({ - engine: "google", - api_key: SERPAPI_TEST_KEY, - q: "Paris", + + it("getHtmlBySearchId (callback)", async () => { + const html = await new Promise< + Awaited> + >((res) => getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); + assertStringIncludes(html, ""); + assertStringIncludes(html, ""); }); - let status; - ({ id, status } = response["search_metadata"]); - assert(id, "Missing search id"); - assertEquals(status, "Success"); - }); - - it("getHtmlBySearchId (async/await)", async () => { - const html = await getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }); - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); - - it("getHtmlBySearchId (callback)", async () => { - const html = await new Promise< - Awaited> - >((res) => getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); -}); + }, +); diff --git a/tests/utils_test.ts b/tests/utils_test.ts index f66b37e..cc23415 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -11,13 +11,27 @@ import { assertInstanceOf, assertMatch, } from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals, buildUrl, execute, getSource } from "../src/utils.ts"; +import { + _internals, + buildRequestOptions, + execute, + getSource, +} from "../src/utils.ts"; import { RequestTimeoutError } from "../src/errors.ts"; +import { Config, config } from "../src/config.ts"; +import http from "node:http"; +import qs from "node:querystring"; loadSync({ export: true }); -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; +const BASE_OPTIONS = Deno.env.get("ENV_TYPE") === "local" + ? { + hostname: "localhost", + port: 3000, + } + : { + hostname: "serpapi.com", + port: 443, + }; describe("getSource", () => { it("use runtime version", async () => { @@ -28,11 +42,11 @@ describe("getSource", () => { }); }); -describe("buildUrl", () => { +describe("buildRequestOptions", () => { let urlStub: Stub; beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); }); afterAll(() => { @@ -40,55 +54,188 @@ describe("buildUrl", () => { }); it("with blank path and empty parameters", async () => { - assertEquals(await buildUrl("", {}), `${BASE_URL}?`); + assertEquals(await buildRequestOptions("", {}), { + ...BASE_OPTIONS, + method: "GET", + path: "?", + }); }); it("with path and empty parameters", async () => { - assertEquals(await buildUrl("/", {}), `${BASE_URL}/?`); + assertEquals(await buildRequestOptions("/", {}), { + ...BASE_OPTIONS, + method: "GET", + path: "/?", + }); }); it("with path and parameters", async () => { assertEquals( - await buildUrl("/search", { q: "coffee", gl: "us" }), - `${BASE_URL}/search?q=coffee&gl=us`, + await buildRequestOptions("/search", { q: "coffee", gl: "us" }), + { + ...BASE_OPTIONS, + method: "GET", + path: "/search?q=coffee&gl=us", + }, ); }); it("with source", async () => { - const url = await buildUrl("/search", { source: await getSource() }); + const options = await buildRequestOptions("/search", { + source: await getSource(), + }); assertMatch( - url, + options.path as string, /source=(nodejs|deno)%40\d+\.\d+\.\d+%2Cserpapi%40\d+\.\d+\.\d+$/, ); }); it("with undefined parameters", async () => { assertEquals( - await buildUrl("/search", { q: "coffee", gl: undefined, hl: null }), - `${BASE_URL}/search?q=coffee&hl=`, + await buildRequestOptions("/search", { + q: "coffee", + gl: undefined, + hl: null, + }), + { + ...BASE_OPTIONS, + method: "GET", + path: "/search?q=coffee&hl=", + }, ); }); -}); -describe("execute", { - sanitizeOps: false, - sanitizeResources: false, -}, () => { - let urlStub: Stub; + describe("with requestOptions", () => { + let originalConfig: Config; - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); + beforeAll(() => { + originalConfig = { ...config }; + }); - afterAll(() => { - urlStub.restore(); - }); + afterAll(() => { + Object.assign(config, originalConfig); + }); + + it("uses default options when no custom options provided", async () => { + const options = await buildRequestOptions("/search", { q: "coffee" }); + assertEquals(options.method, "GET"); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("uses custom request options from parameters", async () => { + const customOptions: http.RequestOptions = { + headers: { + "User-Agent": "Custom User Agent", + "X-Custom-Header": "param-value", + }, + timeout: 5000, + }; + + const params = { + q: "coffee", + requestOptions: customOptions, + } as unknown as qs.ParsedUrlQueryInput; + + const options = await buildRequestOptions("/search", params); + + assertEquals(options.headers?.["User-Agent"], "Custom User Agent"); + assertEquals(options.headers?.["X-Custom-Header"], "param-value"); + assertEquals(options.timeout, 5000); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("uses request options from config when no options in parameters", async () => { + const configOptions: http.RequestOptions = { + headers: { + "User-Agent": "Config User Agent", + "X-Custom-Header": "config-value", + }, + timeout: 5000, + }; - it("with short timeout", async () => { - try { - await execute("/search", { q: "coffee", gl: "us" }, 1); - } catch (e) { - assertInstanceOf(e, RequestTimeoutError); - } + config.requestOptions = configOptions; + + const options = await buildRequestOptions("/search", { q: "coffee" }); + + assertEquals(options.headers?.["User-Agent"], "Config User Agent"); + assertEquals(options.headers?.["X-Custom-Header"], "config-value"); + assertEquals(options.timeout, 5000); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("parameters requestOptions merges over config options", async () => { + const configOptions: http.RequestOptions = { + headers: { + "User-Agent": "Config User Agent", + "X-Custom-Header": "config-value", + }, + timeout: 5000, + }; + + const paramOptions: http.RequestOptions = { + headers: { + "User-Agent": "Parameter User Agent", + "X-Custom-Header": "param-value", + }, + agent: true, + }; + + config.requestOptions = configOptions; + + const params = { + q: "coffee", + requestOptions: paramOptions, + } as unknown as qs.ParsedUrlQueryInput; + + const options = await buildRequestOptions("/search", params); + + assertEquals(options.headers?.["User-Agent"], "Parameter User Agent"); + assertEquals(options.headers?.["X-Custom-Header"], "param-value"); + assertEquals(options.agent, true); + assertEquals(options.timeout, 5000); + assertEquals(options.path, "/search?q=coffee"); + }); + + it("basic options are not allowed to be changed", async () => { + config.requestOptions = { + hostname: "localhost", + port: 3000, + path: "/test", + method: "POST", + }; + + const options = await buildRequestOptions("/search", { q: "coffee" }); + assertEquals(options.hostname, "serpapi.com"); + assertEquals(options.port, 443); + assertEquals(options.path, "/search?q=coffee"); + assertEquals(options.method, "GET"); + }); }); }); + +describe( + "execute", + { + sanitizeOps: false, + sanitizeResources: false, + }, + () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getHostnameAndPort", () => BASE_OPTIONS); + }); + + afterAll(() => { + urlStub.restore(); + }); + + it("with short timeout", async () => { + try { + await execute("/search", { q: "coffee", gl: "us" }, 1); + } catch (e) { + assertInstanceOf(e, RequestTimeoutError); + } + }); + }, +);