fix(search-contexts): Fix issue where a repository would not appear in a search context if it was created after the search context was created (#354)

## Problem

If a repository is added **after** a search context (e.g., a new repository is synced from the code host), then it will never be added to the context even if it should be included. The workaround is to restart the instance.

## Solution

This PR adds a call to re-sync all search contexts whenever a connection is successfully synced. This PR adds the `@sourcebot/shared` package that contains `syncSearchContexts.ts` (previously in web) and it's dependencies (namely the entitlements system).

## Why another package?

Because the `syncSearchContexts` call is now called from:
1. `initialize.ts` in **web** - handles syncing search contexts on startup and whenever the config is modified in watch mode. This is the same as before.
2. `connectionManager.ts` in **backend** - syncs the search contexts whenever a connection is successfully synced.

## Follow-up devex work
Two things:
1. We have several very thin shared packages (i.e., `crypto`, `error`, and `logger`) that we can probably fold into this "general" shared package. `schemas` and `db` _feels_ like they should remain separate (mostly because they are "code-gen" packages).
2. When running `yarn dev`, any changes made to the shared package will only get picked if you `ctrl+c` and restart the instance. Would be nice if we have watch mode work across package dependencies in the monorepo.

Author: brendan-kellam

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Delete account join request when redeeming an invite. [#352](https://github.com/sourcebot-dev/sourcebot/pull/352)
+- Fix issue where a repository would not be included in a search context if the context was created before the repository. [#354](https://github.com/sourcebot-dev/sourcebot/pull/354)
 
 ## [4.3.0] - 2025-06-11
 

Dockerfile

@@ -43,12 +43,14 @@ COPY ./packages/schemas ./packages/schemas
 COPY ./packages/crypto ./packages/crypto
 COPY ./packages/error ./packages/error
 COPY ./packages/logger ./packages/logger
+COPY ./packages/shared ./packages/shared
 
 RUN yarn workspace @sourcebot/db install
 RUN yarn workspace @sourcebot/schemas install
 RUN yarn workspace @sourcebot/crypto install
 RUN yarn workspace @sourcebot/error install
 RUN yarn workspace @sourcebot/logger install
+RUN yarn workspace @sourcebot/shared install
 # ------------------------------------
 
 # ------ Build Web ------
@@ -92,6 +94,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 
 # Fixes arm64 timeouts
 RUN yarn workspace @sourcebot/web install
@@ -132,6 +135,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 RUN yarn workspace @sourcebot/backend install
 RUN yarn workspace @sourcebot/backend build
 
@@ -215,6 +219,7 @@ COPY --from=shared-libs-builder /app/packages/schemas ./packages/schemas
 COPY --from=shared-libs-builder /app/packages/crypto ./packages/crypto
 COPY --from=shared-libs-builder /app/packages/error ./packages/error
 COPY --from=shared-libs-builder /app/packages/logger ./packages/logger
+COPY --from=shared-libs-builder /app/packages/shared ./packages/shared
 
 # Configure dependencies
 RUN apk add --no-cache git ca-certificates bind-tools tini jansson wget supervisor uuidgen curl perl jq redis postgresql postgresql-contrib openssl util-linux unzip

LICENSE

@@ -2,7 +2,7 @@ Copyright (c) 2025 Taqla Inc.
 
 Portions of this software are licensed as follows:
 
-- All content that resides under the "ee/" and "packages/web/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
+- All content that resides under the "ee/", "packages/web/src/ee/", and "packages/shared/src/ee/" directories of this repository, if these directories exist, is licensed under the license defined in "ee/LICENSE".
 - All third party components incorporated into the Sourcebot Software are licensed under the original license provided by the owner of the applicable component.
 - Content outside of the above mentioned directories or restrictions above is available under the "MIT Expat" license as defined below.
 

Makefile

@@ -34,6 +34,8 @@ clean:
 		packages/error/dist \
 		packages/mcp/node_modules \
 		packages/mcp/dist \
+		packages/shared/node_modules \
+		packages/shared/dist \
 		.sourcebot
 
 soft-reset:

package.json

@@ -16,7 +16,7 @@
         "dev:prisma:migrate:dev": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:dev",
         "dev:prisma:studio": "yarn with-env yarn workspace @sourcebot/db prisma:studio",
         "dev:prisma:migrate:reset": "yarn with-env yarn workspace @sourcebot/db prisma:migrate:reset",
-        "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db}' run build"
+        "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build"
     },
     "devDependencies": {
         "cross-env": "^7.0.3",

packages/backend/package.json

@@ -33,9 +33,9 @@
         "@sourcebot/error": "workspace:*",
         "@sourcebot/logger": "workspace:*",
         "@sourcebot/schemas": "workspace:*",
+        "@sourcebot/shared": "workspace:*",
         "@t3-oss/env-core": "^0.12.0",
         "@types/express": "^5.0.0",
-        "ajv": "^8.17.1",
         "argparse": "^2.0.1",
         "bullmq": "^5.34.10",
         "cross-fetch": "^4.0.0",
@@ -50,7 +50,6 @@
         "posthog-node": "^4.2.1",
         "prom-client": "^15.1.3",
         "simple-git": "^3.27.0",
-        "strip-json-comments": "^5.0.1",
         "zod": "^3.24.3"
     }
 }

packages/backend/src/connectionManager.ts

@@ -9,6 +9,7 @@ import { BackendError, BackendException } from "@sourcebot/error";
 import { captureEvent } from "./posthog.js";
 import { env } from "./env.js";
 import * as Sentry from "@sentry/node";
+import { loadConfig, syncSearchContexts } from "@sourcebot/shared";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -264,7 +265,7 @@ export class ConnectionManager implements IConnectionManager {
 
     private async onSyncJobCompleted(job: Job<JobPayload>, result: JobResult) {
         this.logger.info(`Connection sync job for connection ${job.data.connectionName} (id: ${job.data.connectionId}, jobId: ${job.id}) completed`);
-        const { connectionId } = job.data;
+        const { connectionId, orgId } = job.data;
 
         let syncStatusMetadata: Record<string, unknown> = (await this.db.connection.findUnique({
             where: { id: connectionId },
@@ -289,7 +290,25 @@ export class ConnectionManager implements IConnectionManager {
                         notFound.repos.length > 0 ? ConnectionSyncStatus.SYNCED_WITH_WARNINGS : ConnectionSyncStatus.SYNCED,
                 syncedAt: new Date()
             }
-        })
+        });
+
+        // After a connection has synced, we need to re-sync the org's search contexts as
+        // there may be new repos that match the search context's include/exclude patterns.
+        if (env.CONFIG_PATH) {
+            try {
+                const config = await loadConfig(env.CONFIG_PATH);
+
+                await syncSearchContexts({
+                    db: this.db,
+                    orgId,
+                    contexts: config.contexts,
+                });
+            } catch (err) {
+                this.logger.error(`Failed to sync search contexts for connection ${connectionId}: ${err}`);
+                Sentry.captureException(err);
+            }
+        }
+
 
         captureEvent('backend_connection_sync_job_completed', {
             connectionId: connectionId,

packages/backend/src/constants.ts

@@ -16,4 +16,4 @@ export const DEFAULT_SETTINGS: Settings = {
     repoGarbageCollectionGracePeriodMs: 10 * 1000, // 10 seconds
     repoIndexTimeoutMs: 1000 * 60 * 60 * 2, // 2 hours
     enablePublicAccess: false,
-}
+}

packages/backend/src/index.ts

@@ -10,7 +10,8 @@ import { PrismaClient } from "@sourcebot/db";
 import { env } from "./env.js";
 import { createLogger } from "@sourcebot/logger";
 
-const logger = createLogger('index');
+const logger = createLogger('backend-entrypoint');
+
 
 // Register handler for normal exit
 process.on('exit', (code) => {
@@ -72,3 +73,4 @@ main(prisma, context)
     .finally(() => {
         logger.info("Shutting down...");
     });
+

packages/backend/src/main.ts

@@ -7,40 +7,16 @@ import { ConnectionManager } from './connectionManager.js';
 import { RepoManager } from './repoManager.js';
 import { env } from './env.js';
 import { PromClient } from './promClient.js';
-import { isRemotePath } from './utils.js';
-import { readFile } from 'fs/promises';
-import stripJsonComments from 'strip-json-comments';
-import { SourcebotConfig } from '@sourcebot/schemas/v3/index.type';
-import { indexSchema } from '@sourcebot/schemas/v3/index.schema';
-import { Ajv } from "ajv";
+import { loadConfig } from '@sourcebot/shared';
 
 const logger = createLogger('backend-main');
-const ajv = new Ajv({
-    validateFormats: false,
-});
 
 const getSettings = async (configPath?: string) => {
     if (!configPath) {
         return DEFAULT_SETTINGS;
     }
 
-    const configContent = await (async () => {
-        if (isRemotePath(configPath)) {
-            const response = await fetch(configPath);
-            if (!response.ok) {
-                throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
-            }
-            return response.text();
-        } else {
-            return readFile(configPath, { encoding: 'utf-8' });
-        }
-    })();
-
-    const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
-    const isValidConfig = ajv.validate(indexSchema, config);
-    if (!isValidConfig) {
-        throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
-    }
+    const config = await loadConfig(configPath);
 
     return {
         ...DEFAULT_SETTINGS,

packages/backend/src/utils.test.ts

@@ -1,5 +1,6 @@
 import { expect, test } from 'vitest';
-import { arraysEqualShallow, isRemotePath } from './utils';
+import { arraysEqualShallow } from './utils';
+import { isRemotePath } from '@sourcebot/shared';
 
 test('should return true for identical arrays', () => {
     expect(arraysEqualShallow([1, 2, 3], [1, 2, 3])).toBe(true);

packages/backend/src/utils.ts

@@ -20,10 +20,6 @@ export const marshalBool = (value?: boolean) => {
     return !!value ? '1' : '0';
 }
 
-export const isRemotePath = (path: string) => {
-    return path.startsWith('https://') || path.startsWith('http://');
-}
-
 export const getTokenFromConfig = async (token: any, orgId: number, db: PrismaClient, logger?: Logger) => {
     try {
         return await getTokenFromConfigBase(token, orgId, db);

packages/shared/.gitignore

@@ -0,0 +1,2 @@
+dist/
+*.tsbuildinfo 

packages/shared/README.md

@@ -0,0 +1,9 @@
+This package contains shared code between the backend & webapp packages.
+
+### Why two index files?
+
+This package contains two index files: `index.server.ts` and `index.client.ts`. There is some code in this package that will only work in a Node.JS runtime (e.g., because it depends on the `fs` pacakge. Entitlements are a good example of this), and other code that is runtime agnostic (e.g., `constants.ts`). To deal with this, we these two index files export server code and client code, respectively.
+
+For package consumers, the usage would look like the following:
+- Server: `import { ... } from @sourcebot/shared`
+- Client: `import { ... } from @sourcebot/shared/client`

packages/shared/package.json

@@ -0,0 +1,32 @@
+{
+    "name": "@sourcebot/shared",
+    "version": "0.1.0",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "build": "tsc",
+        "build:watch": "tsc-watch --preserveWatchOutput",
+        "postinstall": "yarn build"
+    },
+    "dependencies": {
+        "@sourcebot/crypto": "workspace:*",
+        "@sourcebot/db": "workspace:*",
+        "@sourcebot/logger": "workspace:*",
+        "@sourcebot/schemas": "workspace:*",
+        "@t3-oss/env-core": "^0.12.0",
+        "ajv": "^8.17.1",
+        "micromatch": "^4.0.8",
+        "strip-json-comments": "^5.0.1",
+        "zod": "^3.24.3"
+    },
+    "devDependencies": {
+        "@types/micromatch": "^4.0.9",
+        "@types/node": "^22.7.5",
+        "tsc-watch": "6.2.1",
+        "typescript": "^5.7.3"
+    },
+    "exports": {
+        ".": "./dist/index.server.js",
+        "./client": "./dist/index.client.js"
+    }
+}

packages/shared/src/constants.ts

@@ -0,0 +1,11 @@
+
+export const SOURCEBOT_SUPPORT_EMAIL = '[email protected]';
+
+export const SOURCEBOT_CLOUD_ENVIRONMENT = [
+    "dev",
+    "demo",
+    "staging",
+    "prod",
+] as const;
+
+export const SOURCEBOT_UNLIMITED_SEATS = -1;

packages/web/src/ee/features/searchContexts/syncSearchContexts.ts renamed to packages/shared/src/ee/syncSearchContexts.ts

@@ -1,31 +1,34 @@
-import { env } from "@/env.mjs";
-import { getPlan, hasEntitlement } from "@/features/entitlements/server";
-import { SINGLE_TENANT_ORG_ID, SOURCEBOT_SUPPORT_EMAIL } from "@/lib/constants";
-import { prisma } from "@/prisma";
-import { SearchContext } from "@sourcebot/schemas/v3/index.type";
 import micromatch from "micromatch";
 import { createLogger } from "@sourcebot/logger";
+import { PrismaClient } from "@sourcebot/db";
+import { getPlan, hasEntitlement } from "../entitlements.js";
+import { SOURCEBOT_SUPPORT_EMAIL } from "../constants.js";
+import { SearchContext } from "@sourcebot/schemas/v3/index.type";
 
 const logger = createLogger('sync-search-contexts');
 
-export const syncSearchContexts = async (contexts?: { [key: string]: SearchContext }) => {
-    if (env.SOURCEBOT_TENANCY_MODE !== 'single') {
-        throw new Error("Search contexts are not supported in this tenancy mode. Set SOURCEBOT_TENANCY_MODE=single in your environment variables.");
-    }
+interface SyncSearchContextsParams {
+    contexts?: { [key: string]: SearchContext } | undefined;
+    orgId: number;
+    db: PrismaClient;
+}
+
+export const syncSearchContexts = async (params: SyncSearchContextsParams) => {
+    const { contexts, orgId, db } = params;
 
     if (!hasEntitlement("search-contexts")) {
         if (contexts) {
             const plan = getPlan();
-            logger.error(`Search contexts are not supported in your current plan: ${plan}. If you have a valid enterprise license key, pass it via SOURCEBOT_EE_LICENSE_KEY. For support, contact ${SOURCEBOT_SUPPORT_EMAIL}.`);
+            logger.warn(`Skipping search context sync. Reason: "Search contexts are not supported in your current plan: ${plan}. If you have a valid enterprise license key, pass it via SOURCEBOT_EE_LICENSE_KEY. For support, contact ${SOURCEBOT_SUPPORT_EMAIL}."`);
         }
-        return;
+        return false;
     }
 
     if (contexts) {
         for (const [key, newContextConfig] of Object.entries(contexts)) {
-            const allRepos = await prisma.repo.findMany({
+            const allRepos = await db.repo.findMany({
                 where: {
-                    orgId: SINGLE_TENANT_ORG_ID,
+                    orgId,
                 },
                 select: {
                     id: true,
@@ -44,23 +47,23 @@ export const syncSearchContexts = async (contexts?: { [key: string]: SearchConte
                 });
             }
 
-            const currentReposInContext = (await prisma.searchContext.findUnique({
+            const currentReposInContext = (await db.searchContext.findUnique({
                 where: {
                     name_orgId: {
                         name: key,
-                        orgId: SINGLE_TENANT_ORG_ID,
+                        orgId,
                     }
                 },
                 include: {
                     repos: true,
                 }
             }))?.repos ?? [];
 
-            await prisma.searchContext.upsert({
+            await db.searchContext.upsert({
                 where: {
                     name_orgId: {
                         name: key,
-                        orgId: SINGLE_TENANT_ORG_ID,
+                        orgId,
                     }
                 },
                 update: {
@@ -81,7 +84,7 @@ export const syncSearchContexts = async (contexts?: { [key: string]: SearchConte
                     description: newContextConfig.description,
                     org: {
                         connect: {
-                            id: SINGLE_TENANT_ORG_ID,
+                            id: orgId,
                         }
                     },
                     repos: {
@@ -94,21 +97,23 @@ export const syncSearchContexts = async (contexts?: { [key: string]: SearchConte
         }
     }
 
-    const deletedContexts = await prisma.searchContext.findMany({
+    const deletedContexts = await db.searchContext.findMany({
         where: {
             name: {
                 notIn: Object.keys(contexts ?? {}),
             },
-            orgId: SINGLE_TENANT_ORG_ID,
+            orgId,
         }
     });
 
     for (const context of deletedContexts) {
         logger.info(`Deleting search context with name '${context.name}'. ID: ${context.id}`);
-        await prisma.searchContext.delete({
+        await db.searchContext.delete({
             where: {
                 id: context.id,
             }
         })
     }
+
+    return true;
 }