Gitea support (#45)

Author: brendan-kellam

.github/images/gitea-pat-creation.png

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Gitea support ([#45](https://github.com/sourcebot-dev/sourcebot/pull/45))
+
 ## [2.0.2] - 2024-10-18
 
 ### Added

CHANGELOG.md

@@ -30,7 +30,7 @@ https://github.com/user-attachments/assets/98d46192-5469-430f-ad9e-5c042adbb10d
 
 ## Features
 - 💻 **One-command deployment**: Get started instantly using Docker on your own machine.
-- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub or GitLab.
+- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, or Gitea.
 - ⚡**Lightning fast performance**: Built on top of the powerful [Zoekt](https://github.com/sourcegraph/zoekt) search engine.
 - 📂 **Full file visualization**: Instantly view the entire file when selecting any search result.
 - 🎨 **Modern web app**: Enjoy a sleek interface with features like syntax highlighting, light/dark mode, and vim-style navigation 
@@ -62,7 +62,7 @@ Sourcebot supports indexing and searching through public and private repositorie
 <picture>
     <source media="(prefers-color-scheme: dark)" srcset=".github/images/github-favicon-inverted.png">
     <img src="https://github.com/favicon.ico" width="16" height="16" alt="GitHub icon">
-</picture> GitHub and <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab. This section will guide you through configuring the repositories that Sourcebot indexes. 
+</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab and <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea. This section will guide you through configuring the repositories that Sourcebot indexes. 
 
 1. Create a new folder on your machine that stores your configs and `.sourcebot` cache, and navigate into it:
     ```sh
@@ -214,6 +214,53 @@ docker run -e <b>GITLAB_TOKEN=glpat-mytoken</b> /* additional args */ ghcr.io/so
 
 </details>
 
+<details>
+<summary><img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea</summary>
+
+Generate a Gitea access token [here](http://gitea.com/user/settings/applications). At minimum, you'll need to select the `read:repository` scope, but `read:user` and `read:organization` are required for the `user` and `org` fields of your config file:
+
+![Gitea Access token creation](.github/images/gitea-pat-creation.png)
+
+Next, update your configuration with the `token` field:
+```json
+{
+    "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "gitea",
+            "token": "my-secret-token",
+            ...
+        }
+    ]
+}
+```
+
+You can also pass tokens as environment variables:
+```json
+{
+    "$schema": "https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v2/index.json",
+    "repos": [
+        {
+            "type": "gitea",
+            "token": {
+                // note: this env var can be named anything. It
+                // doesn't need to be `GITEA_TOKEN`.
+                "env": "GITEA_TOKEN"
+            },
+            ...
+        }
+    ]
+}
+```
+
+You'll need to pass this environment variable each time you run Sourcebot:
+
+<pre>
+docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/sourcebot-dev/sourcebot:latest
+</pre>
+
+</details>
+
 </div>
 
 ## Using a self-hosted GitLab / GitHub instance
@@ -226,7 +273,7 @@ If you're using a self-hosted GitLab or GitHub instance with a custom domain, yo
 
 1. Install <a href="https://go.dev/doc/install"><img src="https://go.dev/favicon.ico" width="16" height="16"> go</a> and <a href="https://nodejs.org/"><img src="https://nodejs.org/favicon.ico" width="16" height="16"> NodeJS</a>. Note that a NodeJS version of at least `21.1.0` is required.
 
-2. Install [ctags](https://github.com/universal-ctags/ctags) (required by zoekt-indexserver)
+2. Install [ctags](https://github.com/universal-ctags/ctags) (required by zoekt)
     ```sh
     // macOS:
     brew install universal-ctags

README.md

@@ -22,6 +22,8 @@
         "@gitbeaker/rest": "^40.5.1",
         "@octokit/rest": "^21.0.2",
         "argparse": "^2.0.1",
+        "cross-fetch": "^4.0.0",
+        "gitea-js": "^1.22.0",
         "lowdb": "^7.0.1",
         "simple-git": "^3.27.0",
         "strip-json-comments": "^5.0.1",

packages/backend/package.json

@@ -0,0 +1,150 @@
+import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
+import { GiteaConfig } from './schemas/v2.js';
+import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js';
+import { AppContext, Repository } from './types.js';
+import fetch from 'cross-fetch';
+import { createLogger } from './logger.js';
+import path from 'path';
+
+const logger = createLogger('Gitea');
+
+export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppContext) => {
+    const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined;
+
+    const api = giteaApi(config.url ?? 'https://gitea.com', {
+        token,
+        customFetch: fetch,
+    });
+
+    let allRepos: GiteaRepository[] = [];
+
+    if (config.orgs) {
+        const _repos = await getReposForOrgs(config.orgs, api);
+        allRepos = allRepos.concat(_repos);
+    }
+
+    if (config.repos) {
+        const _repos = await getRepos(config.repos, api);
+        allRepos = allRepos.concat(_repos);
+    }
+
+    if (config.users) {
+        const _repos = await getReposOwnedByUsers(config.users, api);
+        allRepos = allRepos.concat(_repos);
+    }
+
+    let repos: Repository[] = allRepos
+        .map((repo) => {
+            const hostname = config.url ? new URL(config.url).hostname : 'gitea.com';
+            const repoId = `${hostname}/${repo.full_name!}`;
+            const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
+
+            const cloneUrl = new URL(repo.clone_url!);
+            if (token) {
+                cloneUrl.username = token;
+            }
+
+            return {
+                name: repo.full_name!,
+                id: repoId,
+                cloneUrl: cloneUrl.toString(),
+                path: repoPath,
+                isStale: false,
+                isFork: repo.fork!,
+                isArchived: !!repo.archived,
+                gitConfigMetadata: {
+                    'zoekt.web-url-type': 'gitea',
+                    'zoekt.web-url': repo.html_url!,
+                    'zoekt.name': repoId,
+                    'zoekt.archived': marshalBool(repo.archived),
+                    'zoekt.fork': marshalBool(repo.fork!),
+                    'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
+                }
+            } satisfies Repository;
+        });
+    
+    if (config.exclude) {
+        if (!!config.exclude.forks) {
+            repos = excludeForkedRepos(repos, logger);
+        }
+
+        if (!!config.exclude.archived) {
+            repos = excludeArchivedRepos(repos, logger);
+        }
+
+        if (config.exclude.repos) {
+            repos = excludeReposByName(repos, config.exclude.repos, logger);
+        }
+    }
+    
+    return repos;
+}
+
+const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
+    const repos = (await Promise.all(users.map(async (user) => {
+        logger.debug(`Fetching repos for user ${user}...`);
+
+        const { durationMs, data } = await measure(() =>
+            paginate((page) => api.users.userListRepos(user, {
+                page,
+            }))
+        );
+
+        logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
+        return data;
+    }))).flat();
+
+    return repos;
+}
+
+const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
+    return (await Promise.all(orgs.map(async (org) => {
+        logger.debug(`Fetching repos for org ${org}...`);
+
+        const { durationMs, data } = await measure(() =>
+            paginate((page) => api.orgs.orgListRepos(org, {
+                limit: 100,
+                page,
+            }))
+        );
+
+        logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
+        return data;
+    }))).flat();
+}
+
+const getRepos = async <T>(repos: string[], api: Api<T>) => {
+    return Promise.all(repos.map(async (repo) => {
+        logger.debug(`Fetching repository info for ${repo}...`);
+
+        const [owner, repoName] = repo.split('/');
+        const { durationMs, data: response } = await measure(() =>
+            api.repos.repoGet(owner, repoName),
+        );
+
+        logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
+
+        return response.data;
+    }));
+}
+
+// @see : https://docs.gitea.com/development/api-usage#pagination
+const paginate = async <T>(request: (page: number) => Promise<HttpResponse<T[], any>>) => {
+    let page = 1;
+    const result = await request(page);
+    const output: T[] = result.data;
+
+    const totalCountString = result.headers.get('x-total-count');
+    if (!totalCountString) {
+        throw new Error("Header 'x-total-count' not found");
+    }
+    const totalCount = parseInt(totalCountString);
+
+    while (output.length < totalCount) {
+        page++;
+        const result = await request(page);
+        output.push(...result.data);
+    }
+
+    return output;
+}

packages/backend/src/gitea.ts

@@ -6,6 +6,7 @@ import path from 'path';
 import { SourcebotConfigurationSchema } from "./schemas/v2.js";
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
+import { getGiteaReposFromConfig } from "./gitea.js";
 import { AppContext, Repository } from "./types.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
@@ -75,6 +76,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 configRepos.push(...gitLabRepos);
                 break;
             }
+            case 'gitea': {
+                const giteaRepos = await getGiteaReposFromConfig(repoConfig, ctx);
+                configRepos.push(...giteaRepos);
+                break;
+            }
         }
     }
 
@@ -180,7 +186,8 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                     // since it implies another sync is in progress.
                 } else {
                     isSyncing = false;
-                    logger.error(`Failed to sync configuration file ${args.configPath} with error:\n`, err);
+                    logger.error(`Failed to sync configuration file ${args.configPath} with error:`);
+                    console.log(err);
                 }
             });
     }

packages/backend/src/index.ts

@@ -1,6 +1,6 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type Repos = GitHubConfig | GitLabConfig;
+export type Repos = GitHubConfig | GitLabConfig | GiteaConfig;
 
 /**
  * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -106,3 +106,50 @@ export interface GitLabConfig {
     projects?: string[];
   };
 }
+export interface GiteaConfig {
+  /**
+   * Gitea Configuration
+   */
+  type: "gitea";
+  /**
+   * An access token.
+   */
+  token?:
+    | string
+    | {
+        /**
+         * The name of the environment variable that contains the token.
+         */
+        env: string;
+      };
+  /**
+   * The URL of the Gitea host. Defaults to https://gitea.com
+   */
+  url?: string;
+  /**
+   * List of organizations to sync with. All repositories in the organization visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:organization scope.
+   */
+  orgs?: string[];
+  /**
+   * List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'.
+   */
+  repos?: string[];
+  /**
+   * List of users to sync with. All repositories that the user owns will be synced, unless explicitly defined in the `exclude` property. If a `token` is provided, it must have the read:user scope.
+   */
+  users?: string[];
+  exclude?: {
+    /**
+     * Exlcude forked repositories from syncing.
+     */
+    forks?: boolean;
+    /**
+     * Exlcude archived repositories from syncing.
+     */
+    archived?: boolean;
+    /**
+     * List of individual repositories to exclude from syncing. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'.
+     */
+    repos?: string[];
+  };
+}

packages/backend/src/schemas/v2.ts

@@ -64,7 +64,7 @@ const RepositoryBadge = ({
                 repoIcon: <Image
                     src={info.icon}
                     alt={info.costHostName}
-                    className="w-4 h-4 dark:invert"
+                    className={`w-4 h-4 ${info.iconClassname}`}
                 />,
                 repoName: info.repoName,
                 repoLink: info.repoLink,

packages/web/public/gitea.svg

@@ -63,7 +63,7 @@ export const FileMatchContainer = ({
                 repoIcon: <Image
                     src={info.icon}
                     alt={info.costHostName}
-                    className="w-4 h-4 dark:invert"
+                    className={`w-4 h-4 ${info.iconClassname}`}
                 />
             }
         }