From 3bda28a1310f77407bb807d9be66033d15236e77 Mon Sep 17 00:00:00 2001 From: j-rewerts Date: Tue, 22 Oct 2019 10:04:22 -0600 Subject: [PATCH 1/5] Added paged results to search tests. --- test/fixtures/search.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/fixtures/search.json b/test/fixtures/search.json index e87ad025..9b73c28b 100644 --- a/test/fixtures/search.json +++ b/test/fixtures/search.json @@ -8911,7 +8911,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:11 GMT", "content-type": "application/json; charset=utf-8", @@ -8939,7 +8939,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=2&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100", + "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=2&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=2", "body": "", "status": 200, "response": { @@ -17848,7 +17848,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:14 GMT", "content-type": "application/json; charset=utf-8", @@ -17876,7 +17876,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=3&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100", + "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=3&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=3", "body": "", "status": 200, "response": { @@ -26785,7 +26785,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:17 GMT", "content-type": "application/json; charset=utf-8", @@ -26813,7 +26813,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=4&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100", + "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=4&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=4", "body": "", "status": 200, "response": { @@ -35722,7 +35722,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:20 GMT", "content-type": "application/json; charset=utf-8", @@ -35750,7 +35750,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=5&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100", + "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=5&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=5", "body": "", "status": 200, "response": { @@ -42256,7 +42256,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:22 GMT", "content-type": "application/json; charset=utf-8", @@ -42885,7 +42885,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:23 GMT", "content-type": "application/json; charset=utf-8", @@ -48918,7 +48918,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:24 GMT", "content-type": "application/json; charset=utf-8", @@ -48946,7 +48946,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/issues?q=windows+pip+label%3Abug+language%3Apython+state%3Aopen+&sort=created&order=asc&type=all&per_page=100&page=2&q=windows+pip+label:bug+language:python+state:open+&sort=created&order=asc&type=all&per_page=100", + "path": "/search/issues?q=windows+pip+label%3Abug+language%3Apython+state%3Aopen+&sort=created&order=asc&type=all&per_page=100&page=2&q=windows+pip+label:bug+language:python+state:open+&sort=created&order=asc&type=all&per_page=100&page=2", "body": "", "status": 200, "response": { @@ -53447,7 +53447,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:26 GMT", "content-type": "application/json; charset=utf-8", @@ -53564,7 +53564,7 @@ } ] }, - "headers": { + "rawHeaders": { "server": "GitHub.com", "date": "Wed, 22 Jun 2016 02:15:27 GMT", "content-type": "application/json; charset=utf-8", From fb4cc02cdf0c21baa89c48643f1a5a68e4ea5f73 Mon Sep 17 00:00:00 2001 From: j-rewerts Date: Tue, 22 Oct 2019 10:07:57 -0600 Subject: [PATCH 2/5] Assert the exact response length of all search tests. --- test/search.spec.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/search.spec.js b/test/search.spec.js index 11614e4e..015da341 100644 --- a/test/search.spec.js +++ b/test/search.spec.js @@ -29,6 +29,7 @@ describe('Search', function() { .then(function({data}) { expect(data).to.be.an.array(); expect(data.length).to.be.above(0); + expect(data.length).to.be.equal(473); }); }); @@ -42,6 +43,7 @@ describe('Search', function() { .then(function({data}) { expect(data).to.be.an.array(); expect(data.length).to.be.above(0); + expect(data.length).to.be.equal(8); }); }); @@ -57,6 +59,7 @@ describe('Search', function() { .then(function({data}) { expect(data).to.be.an.array(); expect(data.length).to.be.above(0); + expect(data.length).to.be.equal(161); }); }); @@ -70,6 +73,7 @@ describe('Search', function() { .then(function({data}) { expect(data).to.be.an.array(); expect(data.length).to.be.above(0); + expect(data.length).to.be.equal(4); }); }); From cd8c27fe088c108db7b00ce70de1295b9789f22e Mon Sep 17 00:00:00 2001 From: j-rewerts Date: Tue, 22 Oct 2019 10:55:38 -0600 Subject: [PATCH 3/5] Fixed error handling. --- lib/Requestable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Requestable.js b/lib/Requestable.js index 4d6e8d9d..8685f4de 100644 --- a/lib/Requestable.js +++ b/lib/Requestable.js @@ -306,7 +306,7 @@ function getNextPage(linksHeader = '') { function callbackErrorOrThrow(cb, path) { return function handler(object) { let error; - if (object.hasOwnProperty('config')) { + if (object.hasOwnProperty('config') && object.response) { const {response: {status, statusText}, config: {method, url}} = object; let message = (`${status} error making request ${method} ${url}: "${statusText}"`); error = new ResponseError(message, path, object); From 5718843977177673683d3273d588ac46f3e23869 Mon Sep 17 00:00:00 2001 From: j-rewerts Date: Tue, 22 Oct 2019 10:57:59 -0600 Subject: [PATCH 4/5] Fixed duplicate params when paging. --- lib/Requestable.js | 4 +++- test/fixtures/search.json | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/Requestable.js b/lib/Requestable.js index 8685f4de..9a2c28d5 100644 --- a/lib/Requestable.js +++ b/lib/Requestable.js @@ -255,7 +255,7 @@ class Requestable { } results.push(...thisGroup); - const nextUrl = getNextPage(response.headers.link); + let nextUrl = getNextPage(response.headers.link); if(nextUrl) { if (!options) { options = {}; @@ -266,6 +266,8 @@ class Requestable { .split('=') .pop() ); + // Strip out query string, as we'll use 'options'. + nextUrl = nextUrl.split('?', 1)[0]; if (!(options && typeof options.page !== 'number')) { log(`getting next page: ${nextUrl}`); return this._requestAllPages(nextUrl, options, cb, results); diff --git a/test/fixtures/search.json b/test/fixtures/search.json index 9b73c28b..9a599a79 100644 --- a/test/fixtures/search.json +++ b/test/fixtures/search.json @@ -8939,7 +8939,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=2&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=2", + "path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=2", "body": "", "status": 200, "response": { @@ -17876,7 +17876,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=3&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=3", + "path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=3", "body": "", "status": 200, "response": { @@ -26813,7 +26813,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=4&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=4", + "path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=4", "body": "", "status": 200, "response": { @@ -35750,7 +35750,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/repositories?q=tetris+language%3Aassembly&sort=stars&order=desc&type=all&per_page=100&page=5&q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=5", + "path": "/search/repositories?q=tetris+language:assembly&sort=stars&order=desc&type=all&per_page=100&page=5", "body": "", "status": 200, "response": { @@ -48946,7 +48946,7 @@ { "scope": "https://api.github.com:443", "method": "GET", - "path": "/search/issues?q=windows+pip+label%3Abug+language%3Apython+state%3Aopen+&sort=created&order=asc&type=all&per_page=100&page=2&q=windows+pip+label:bug+language:python+state:open+&sort=created&order=asc&type=all&per_page=100&page=2", + "path": "/search/issues?q=windows+pip+label:bug+language:python+state:open+&sort=created&order=asc&type=all&per_page=100&page=2", "body": "", "status": 200, "response": { From 8b54caf1964f2510a2e4ad3fa4c088960b318f15 Mon Sep 17 00:00:00 2001 From: j-rewerts Date: Tue, 22 Oct 2019 12:19:47 -0600 Subject: [PATCH 5/5] Added single page search. --- lib/Requestable.js | 38 ++++++++++++++++++++++++++++++++++++++ lib/Search.js | 6 +++++- test/search.spec.js | 20 ++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) diff --git a/lib/Requestable.js b/lib/Requestable.js index 9a2c28d5..389947c2 100644 --- a/lib/Requestable.js +++ b/lib/Requestable.js @@ -282,6 +282,44 @@ class Requestable { return response; }).catch(callbackErrorOrThrow(cb, path)); } + + /** + * Make a request and fetch a specific page. Use this to override the paging provided + * by {@link Requestable#requestAllPages}. + * @param {string} path - the path to request + * @param {string} perPage - the maximum number of results to return + * @param {string} page - the page number + * @param {Object} options - the query parameters to include. It doesn't matter if this includes 'per_page' or + * 'page'. + * @param {Requestable.callback} [cb] - the function to receive the data. The returned data will always be an array. + * @return {Promise} - a promise which will resolve the page has been fetched + */ + _requestPage(path, perPage, page, options, cb) { + options['page'] = page; + options['per_page'] = perPage; + + let results = []; + return this._request('GET', path, options) + .then((response) => { + let thisGroup; + if (response.data instanceof Array) { + thisGroup = response.data; + } else if (response.data.items instanceof Array) { + thisGroup = response.data.items; + } else { + let message = `cannot figure out how to append ${response.data} to the result set`; + throw new ResponseError(message, path, response); + } + results.push(...thisGroup); + + if (cb) { + cb(null, results, response); + } + + response.data = results; + return response; + }).catch(callbackErrorOrThrow(cb, path)); + } } module.exports = Requestable; diff --git a/lib/Search.js b/lib/Search.js index e0bde4fb..2f9d69fd 100644 --- a/lib/Search.js +++ b/lib/Search.js @@ -51,7 +51,11 @@ class Search extends Requestable { }); log(`searching ${path} with options:`, requestOptions); - return this._requestAllPages(`/search/${path}`, requestOptions, cb); + if (requestOptions.page && requestOptions.per_page) { + return this._requestPage(`/search/${path}`, requestOptions.per_page, requestOptions.page, requestOptions, cb); + } else { + return this._requestAllPages(`/search/${path}`, requestOptions, cb); + } } /** diff --git a/test/search.spec.js b/test/search.spec.js index 015da341..21a3ec1e 100644 --- a/test/search.spec.js +++ b/test/search.spec.js @@ -33,6 +33,26 @@ describe('Search', function() { }); }); + it('should search repositories for specific page', function() { + let options; + let search = github.search({ + q: 'tetris language:assembly', + sort: 'stars', + order: 'desc', + type: 'all', + per_page: '100', + page: '2', + }); + + return search.forRepositories(options) + .then(function({data}) { + expect(data).to.be.an.array(); + expect(data.length).to.be.above(0); + expect(data.length).to.be.equal(100); + expect(data[0].name).to.be.equal('nand2tetris'); + }); + }); + it('should search code', function() { let options; let search = github.search({