Skip to content

Commit 03aa608

Browse files
brendan-kellambrendan-kellam
authored
Graceful error handling when calling code host apis (#142)
1 parent 4e68dc5 commit 03aa608

File tree

9 files changed

+276
-202
lines changed

9 files changed

+276
-202
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1212
- Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))
1313
- Added `exclude.size` to the GitHub config to allow excluding repositories by size. ([#137](https://github.com/sourcebot-dev/sourcebot/pull/137))
1414

15+
### Fixed
16+
17+
- Fixed issue where config synchronization was failing entirely when a single api call fails. ([#142](https://github.com/sourcebot-dev/sourcebot/pull/142))
18+
1519
## [2.6.2] - 2024-12-13
1620

1721
### Added

‎packages/backend/src/gerrit.ts

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,18 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
2828
const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
2929
const hostname = new URL(config.url).hostname;
3030

31-
const { durationMs, data: projects } = await measure(() =>
32-
fetchAllProjects(url)
33-
);
31+
const { durationMs, data: projects } = await measure(async () => {
32+
try {
33+
return fetchAllProjects(url)
34+
} catch (err) {
35+
logger.error(`Failed to fetch projects from ${url}`, err);
36+
return null;
37+
}
38+
});
39+
40+
if (!projects) {
41+
return [];
42+
}
3443

3544
// exclude "All-Projects" and "All-Users" projects
3645
delete projects['All-Projects'];

‎packages/backend/src/gitea.ts

Lines changed: 71 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -122,73 +122,98 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
122122
}
123123

124124
const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
125-
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
126-
const { durationMs, data: tags } = await measure(() =>
127-
paginate((page) => api.repos.repoListTags(owner, repo, {
128-
page
129-
}))
130-
);
131-
logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
132-
return tags;
125+
try {
126+
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
127+
const { durationMs, data: tags } = await measure(() =>
128+
paginate((page) => api.repos.repoListTags(owner, repo, {
129+
page
130+
}))
131+
);
132+
logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
133+
return tags;
134+
} catch (e) {
135+
logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
136+
return [];
137+
}
133138
}
134139

135140
const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
136-
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
137-
const { durationMs, data: branches } = await measure(() =>
138-
paginate((page) => api.repos.repoListBranches(owner, repo, {
139-
page
140-
}))
141-
);
142-
logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
143-
return branches;
141+
try {
142+
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
143+
const { durationMs, data: branches } = await measure(() =>
144+
paginate((page) => api.repos.repoListBranches(owner, repo, {
145+
page
146+
}))
147+
);
148+
logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
149+
return branches;
150+
} catch (e) {
151+
logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
152+
return [];
153+
}
144154
}
145155

146156
const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
147157
const repos = (await Promise.all(users.map(async (user) => {
148-
logger.debug(`Fetching repos for user ${user}...`);
149-
150-
const { durationMs, data } = await measure(() =>
151-
paginate((page) => api.users.userListRepos(user, {
152-
page,
153-
}))
154-
);
155-
156-
logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
157-
return data;
158+
try {
159+
logger.debug(`Fetching repos for user ${user}...`);
160+
161+
const { durationMs, data } = await measure(() =>
162+
paginate((page) => api.users.userListRepos(user, {
163+
page,
164+
}))
165+
);
166+
167+
logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
168+
return data;
169+
} catch (e) {
170+
logger.error(`Failed to fetch repos for user ${user}.`, e);
171+
return [];
172+
}
158173
}))).flat();
159174

160175
return repos;
161176
}
162177

163178
const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
164179
return (await Promise.all(orgs.map(async (org) => {
165-
logger.debug(`Fetching repos for org ${org}...`);
166-
167-
const { durationMs, data } = await measure(() =>
168-
paginate((page) => api.orgs.orgListRepos(org, {
169-
limit: 100,
170-
page,
171-
}))
172-
);
173-
174-
logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
175-
return data;
180+
try {
181+
logger.debug(`Fetching repos for org ${org}...`);
182+
183+
const { durationMs, data } = await measure(() =>
184+
paginate((page) => api.orgs.orgListRepos(org, {
185+
limit: 100,
186+
page,
187+
}))
188+
);
189+
190+
logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
191+
return data;
192+
} catch (e) {
193+
logger.error(`Failed to fetch repos for org ${org}.`, e);
194+
return [];
195+
}
176196
}))).flat();
177197
}
178198

179199
const getRepos = async <T>(repos: string[], api: Api<T>) => {
180-
return Promise.all(repos.map(async (repo) => {
181-
logger.debug(`Fetching repository info for ${repo}...`);
200+
return (await Promise.all(repos.map(async (repo) => {
201+
try {
202+
logger.debug(`Fetching repository info for ${repo}...`);
182203

183-
const [owner, repoName] = repo.split('/');
184-
const { durationMs, data: response } = await measure(() =>
185-
api.repos.repoGet(owner, repoName),
186-
);
204+
const [owner, repoName] = repo.split('/');
205+
const { durationMs, data: response } = await measure(() =>
206+
api.repos.repoGet(owner, repoName),
207+
);
187208

188-
logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
209+
logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
189210

190-
return response.data;
191-
}));
211+
return [response.data];
212+
} catch (e) {
213+
logger.error(`Failed to fetch repository info for ${repo}.`, e);
214+
return [];
215+
}
216+
}))).flat();
192217
}
193218

194219
// @see : https://docs.gitea.com/development/api-usage#pagination

‎packages/backend/src/github.ts

Lines changed: 101 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -201,114 +201,129 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
201201
}
202202

203203
const getTagsForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
204-
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
205-
206-
const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
207-
owner,
208-
repo,
209-
per_page: 100,
210-
request: {
211-
signal
212-
}
213-
}));
204+
try {
205+
logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
206+
const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
207+
owner,
208+
repo,
209+
per_page: 100,
210+
request: {
211+
signal
212+
}
213+
}));
214214

215-
logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
216-
return tags;
215+
logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
216+
return tags;
217+
} catch (e) {
218+
logger.debug(`Error fetching tags for repo ${owner}/${repo}: ${e}`);
219+
return [];
220+
}
217221
}
218222

219223
const getBranchesForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
220-
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
221-
const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
222-
owner,
223-
repo,
224-
per_page: 100,
225-
request: {
226-
signal
227-
}
228-
}));
229-
logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
230-
return branches;
224+
try {
225+
logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
226+
const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
227+
owner,
228+
repo,
229+
per_page: 100,
230+
request: {
231+
signal
232+
}
233+
}));
234+
logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
235+
return branches;
236+
} catch (e) {
237+
logger.debug(`Error fetching branches for repo ${owner}/${repo}: ${e}`);
238+
return [];
239+
}
231240
}
232241

233242

234243
const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, octokit: Octokit, signal: AbortSignal) => {
235-
// @todo : error handling
236244
const repos = (await Promise.all(users.map(async (user) => {
237-
logger.debug(`Fetching repository info for user ${user}...`);
238-
const start = Date.now();
239-
240-
const result = await (() => {
241-
if (isAuthenticated) {
242-
return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
243-
username: user,
244-
visibility: 'all',
245-
affiliation: 'owner',
246-
per_page: 100,
247-
request: {
248-
signal,
249-
},
250-
});
251-
} else {
252-
return octokit.paginate(octokit.repos.listForUser, {
253-
username: user,
254-
per_page: 100,
255-
request: {
256-
signal,
257-
},
258-
});
259-
}
260-
})();
261-
262-
const duration = Date.now() - start;
263-
logger.debug(`Found ${result.length} owned by user ${user} in ${duration}ms.`);
264-
265-
return result;
245+
try {
246+
logger.debug(`Fetching repository info for user ${user}...`);
247+
248+
const { durationMs, data } = await measure(async () => {
249+
if (isAuthenticated) {
250+
return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
251+
username: user,
252+
visibility: 'all',
253+
affiliation: 'owner',
254+
per_page: 100,
255+
request: {
256+
signal,
257+
},
258+
});
259+
} else {
260+
return octokit.paginate(octokit.repos.listForUser, {
261+
username: user,
262+
per_page: 100,
263+
request: {
264+
signal,
265+
},
266+
});
267+
}
268+
});
269+
270+
logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`);
271+
return data;
272+
} catch (e) {
273+
logger.error(`Failed to fetch repository info for user ${user}.`, e);
274+
return [];
275+
}
266276
}))).flat();
267277

268278
return repos;
269279
}
270280

271281
const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
272282
const repos = (await Promise.all(orgs.map(async (org) => {
273-
logger.debug(`Fetching repository info for org ${org}...`);
274-
const start = Date.now();
275-
276-
const result = await octokit.paginate(octokit.repos.listForOrg, {
277-
org: org,
278-
per_page: 100,
279-
request: {
280-
signal
281-
}
282-
});
283-
284-
const duration = Date.now() - start;
285-
logger.debug(`Found ${result.length} in org ${org} in ${duration}ms.`);
286-
287-
return result;
283+
try {
284+
logger.debug(`Fetching repository info for org ${org}...`);
285+
286+
const { durationMs, data } = await measure(() => octokit.paginate(octokit.repos.listForOrg, {
287+
org: org,
288+
per_page: 100,
289+
request: {
290+
signal
291+
}
292+
}));
293+
294+
logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
295+
return data;
296+
} catch (e) {
297+
logger.error(`Failed to fetch repository info for org ${org}.`, e);
298+
return [];
299+
}
288300
}))).flat();
289301

290302
return repos;
291303
}
292304

293305
const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
294-
const repos = await Promise.all(repoList.map(async (repo) => {
295-
logger.debug(`Fetching repository info for ${repo}...`);
296-
const start = Date.now();
297-
298-
const [owner, repoName] = repo.split('/');
299-
const result = await octokit.repos.get({
300-
owner,
301-
repo: repoName,
302-
request: {
303-
signal
304-
}
305-
});
306-
307-
const duration = Date.now() - start;
308-
logger.debug(`Found info for repository ${repo} in ${duration}ms`);
309-
310-
return result.data;
311-
}));
306+
const repos = (await Promise.all(repoList.map(async (repo) => {
307+
try {
308+
logger.debug(`Fetching repository info for ${repo}...`);
309+
310+
const [owner, repoName] = repo.split('/');
311+
const { durationMs, data: result } = await measure(() => octokit.repos.get({
312+
owner,
313+
repo: repoName,
314+
request: {
315+
signal
316+
}
317+
}));
318+
319+
logger.debug(`Found info for repository ${repo} in ${durationMs}ms`);
320+
321+
return [result.data];
322+
} catch (e) {
323+
logger.error(`Failed to fetch repository info for ${repo}.`, e);
324+
return [];
325+
}
326+
}))).flat();
312327

313328
return repos;
314329
}

0 commit comments

Comments
 (0)