Merge branch 'main' into dallin/mcp-history

Author: sestinj

CONTRIBUTING.md

@@ -26,7 +26,6 @@
     - [Writing Context Providers](#writing-context-providers)
     - [Adding an LLM Provider](#adding-an-llm-provider)
     - [Adding Models](#adding-models)
-    - [Adding Pre-indexed Documentation](#adding-pre-indexed-documentation)
   - [📐 Continue Architecture](#-continue-architecture)
     - [Continue VS Code Extension](#continue-vs-code-extension)
     - [Continue JetBrains Extension](#continue-jetbrains-extension)
@@ -212,10 +211,6 @@ While any model that works with a supported provider can be used with Continue,
 - LLM Providers: Since many providers use their own custom strings to identify models, you'll have to add the translation from Continue's model name (the one you added to `index.d.ts`) and the model string for each of these providers: [Ollama](./core/llm/llms/Ollama.ts), [Together](./core/llm/llms/Together.ts), and [Replicate](./core/llm/llms/Replicate.ts). You can find their full model lists here: [Ollama](https://ollama.ai/library), [Together](https://docs.together.ai/docs/inference-models), [Replicate](https://replicate.com/collections/streaming-language-models).
 - [Prompt Templates](./core/llm/index.ts) - In this file you'll find the `autodetectTemplateType` function. Make sure that for the model name you just added, this function returns the correct template type. This is assuming that the chat template for that model is already built in Continue. If not, you will have to add the template type and corresponding edit and chat templates.
 
-### Adding Pre-indexed Documentation
-
-Continue's @docs context provider lets you easily reference entire documentation sites and then uses embeddings to add the most relevant pages to context. To make the experience as smooth as possible, we pre-index many of the most popular documentation sites. If you'd like to add new documentation to this list, just add an object to the list in [preIndexedDocs.ts](./core/indexing/docs/preIndexedDocs.ts). `startUrl` is where the crawler will start and `rootUrl` will filter out any pages not on that site and under the path of `rootUrl`.
-
 ## 📐 Continue Architecture
 
 Continue consists of 2 parts that are split so that it can be extended to work in other IDEs as easily as possible:

binary/build.js

@@ -243,7 +243,7 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
     fs.mkdirSync(targetDir, { recursive: true });
     console.log(`[info] Building ${target}...`);
     execCmdSync(
-      `npx pkg --no-bytecode --public-packages "*" --public pkgJson/${target} --out-path ${targetDir}`,
+      `npx pkg --no-bytecode --public-packages "*" --public --compress GZip pkgJson/${target} --out-path ${targetDir}`,
     );
 
     // Download and unzip prebuilt sqlite3 binary for the target

core/.eslintrc.json

@@ -14,7 +14,7 @@
     "eqeqeq": "error",
     "complexity": ["error", { "max": 38 }],
     "max-lines-per-function": ["error", { "max": 996 }],
-    "max-statements": ["error", { "max": 112 }],
+    "max-statements": ["error", { "max": 114 }],
     "max-depth": ["error", { "max": 6 }],
     "max-nested-callbacks": ["error", { "max": 4 }],
     "max-params": ["error", { "max": 11 }]

core/config/ConfigHandler.ts

@@ -199,8 +199,8 @@ export class ConfigHandler {
             this.ide,
             this.ideSettingsPromise,
             this.writeLog,
-            this.reloadConfig.bind(this),
             assistant.rawYaml,
+            selectedOrgId,
           );
 
           return new ProfileLifecycleManager(profileLoader, this.ide);

core/config/profile/ControlPlaneProfileLoader.ts

@@ -71,6 +71,7 @@ export default class ControlPlaneProfileLoader implements IProfileLoader {
       undefined,
       this.workspaceId,
       undefined,
+      null,
     );
   }
 

core/config/profile/LocalProfileLoader.ts

@@ -65,6 +65,7 @@ export default class LocalProfileLoader implements IProfileLoader {
       undefined,
       this.description.id,
       this.overrideAssistantFile?.path,
+      null,
     );
 
     this.description.errors = result.errors;

core/config/profile/PlatformProfileLoader.ts

@@ -31,8 +31,8 @@ export default class PlatformProfileLoader implements IProfileLoader {
     private readonly ide: IDE,
     private ideSettingsPromise: Promise<IdeSettings>,
     private writeLog: (message: string) => Promise<void>,
-    private readonly onReload: () => void,
     readonly description: ProfileDescription,
+    private readonly orgScopeId: string | null,
   ) {}
 
   static async create(
@@ -45,8 +45,8 @@ export default class PlatformProfileLoader implements IProfileLoader {
     ide: IDE,
     ideSettingsPromise: Promise<IdeSettings>,
     writeLog: (message: string) => Promise<void>,
-    onReload: () => void,
     rawYaml: string,
+    orgScopeId: string | null,
   ): Promise<PlatformProfileLoader> {
     const controlPlaneEnv = await getControlPlaneEnv(ideSettingsPromise);
 
@@ -75,8 +75,8 @@ export default class PlatformProfileLoader implements IProfileLoader {
       ide,
       ideSettingsPromise,
       writeLog,
-      onReload,
       description,
+      orgScopeId,
     );
   }
 
@@ -102,6 +102,7 @@ export default class PlatformProfileLoader implements IProfileLoader {
       },
       this.description.id,
       undefined,
+      this.orgScopeId,
     );
 
     return {

core/config/profile/doLoadConfig.ts

@@ -45,6 +45,7 @@ export default async function doLoadConfig(
   platformConfigMetadata: PlatformConfigMetadata | undefined,
   profileId: string,
   overrideConfigYamlByPath: string | undefined,
+  orgScopeId: string | null,
 ): Promise<ConfigResult<ContinueConfig>> {
   const workspaceConfigs = await getWorkspaceConfigs(ide);
   const ideInfo = await ide.getIdeInfo();
@@ -70,7 +71,6 @@ export default async function doLoadConfig(
   if (overrideConfigYaml || fs.existsSync(configYamlPath)) {
     const result = await loadContinueConfigFromYaml(
       ide,
-      workspaceConfigs.map((c) => JSON.stringify(c)),
       ideSettings,
       ideInfo,
       uniqueId,
@@ -80,6 +80,7 @@ export default async function doLoadConfig(
       platformConfigMetadata,
       controlPlaneClient,
       configYamlPath,
+      orgScopeId,
     );
     newConfig = result.config;
     errors = result.errors;

core/config/yaml/loadYaml.ts

@@ -41,22 +41,31 @@ import { getSystemPromptDotFile } from "../getSystemPromptDotFile";
 import { PlatformConfigMetadata } from "../profile/PlatformProfileLoader";
 import { modifyAnyConfigWithSharedConfig } from "../sharedConfig";
 
+import { getControlPlaneEnvSync } from "../../control-plane/env";
 import { llmsFromModelConfig } from "./models";
 
 export class LocalPlatformClient implements PlatformClient {
+  constructor(
+    private orgScopeId: string | null,
+    private readonly client: ControlPlaneClient,
+  ) {}
+
   async resolveFQSNs(fqsns: FQSN[]): Promise<(SecretResult | undefined)[]> {
-    return fqsns.map((fqsn) => {
-      return undefined;
-    });
+    if (fqsns.length === 0) {
+      return [];
+    }
+
+    const response = await this.client.resolveFQSNs(fqsns, this.orgScopeId);
+    return response;
   }
 }
 
 async function loadConfigYaml(
-  workspaceConfigs: string[],
   rawYaml: string,
   overrideConfigYaml: AssistantUnrolled | undefined,
-  ide: IDE,
   controlPlaneClient: ControlPlaneClient,
+  orgScopeId: string | null,
+  ideSettings: IdeSettings,
 ): Promise<ConfigResult<AssistantUnrolled>> {
   let config =
     overrideConfigYaml ??
@@ -68,12 +77,18 @@ async function loadConfigYaml(
         versionSlug: "",
       },
       rawYaml,
-      new RegistryClient(),
+      new RegistryClient(
+        await controlPlaneClient.getAccessToken(),
+        getControlPlaneEnvSync(
+          ideSettings.continueTestEnvironment,
+          ideSettings.enableControlServerBeta,
+        ).CONTROL_PLANE_URL,
+      ),
       {
         currentUserSlug: "",
         onPremProxyUrl: null,
-        orgScopeId: null,
-        platformClient: new LocalPlatformClient(),
+        orgScopeId,
+        platformClient: new LocalPlatformClient(orgScopeId, controlPlaneClient),
         renderSecrets: true,
       },
     ));
@@ -206,7 +221,6 @@ async function configYamlToContinueConfig(
         continueConfig.systemMessage,
       );
 
-      //
       if (modelsArrayRoles.some((role) => model.roles?.includes(role))) {
         continueConfig.models.push(...llms);
       }
@@ -244,7 +258,10 @@ async function configYamlToContinueConfig(
           } else {
             continueConfig.modelsByRole.embed.push(
               new embeddingsProviderClass(
-                options,
+                {
+                  ...options,
+                  title: options.name,
+                },
                 (url: string | URL, init: any) =>
                   fetchwithRequestOptions(url, init, {
                     ...options.requestOptions,
@@ -265,10 +282,15 @@ async function configYamlToContinueConfig(
         const rerankerClass = AllRerankers[provider];
         if (rerankerClass) {
           continueConfig.modelsByRole.rerank.push(
-            new rerankerClass(options, (url: string | URL, init: any) =>
-              fetchwithRequestOptions(url, init, {
-                ...options.requestOptions,
-              }),
+            new rerankerClass(
+              {
+                ...options,
+                title: options.name,
+              },
+              (url: string | URL, init: any) =>
+                fetchwithRequestOptions(url, init, {
+                  ...options.requestOptions,
+                }),
             ),
           );
         } else {
@@ -387,7 +409,6 @@ async function configYamlToContinueConfig(
 
 export async function loadContinueConfigFromYaml(
   ide: IDE,
-  workspaceConfigs: string[],
   ideSettings: IdeSettings,
   ideInfo: IdeInfo,
   uniqueId: string,
@@ -397,6 +418,7 @@ export async function loadContinueConfigFromYaml(
   platformConfigMetadata: PlatformConfigMetadata | undefined,
   controlPlaneClient: ControlPlaneClient,
   configYamlPath: string | undefined,
+  orgScopeId: string | null,
 ): Promise<ConfigResult<ContinueConfig>> {
   const rawYaml =
     overrideConfigYaml === undefined
@@ -407,11 +429,11 @@ export async function loadContinueConfigFromYaml(
       : "";
 
   const configYamlResult = await loadConfigYaml(
-    workspaceConfigs,
     rawYaml,
     overrideConfigYaml,
-    ide,
     controlPlaneClient,
+    orgScopeId,
+    ideSettings,
   );
 
   if (!configYamlResult.config || configYamlResult.configLoadInterrupted) {

core/context/providers/DocsContextProvider.ts

@@ -9,7 +9,6 @@ import {
   LoadSubmenuItemsArgs,
 } from "../..";
 import DocsService from "../../indexing/docs/DocsService";
-import preIndexedDocs from "../../indexing/docs/preIndexedDocs";
 
 import { INSTRUCTIONS_BASE_ITEM } from "./utils";
 
@@ -64,23 +63,12 @@ class DocsContextProvider extends BaseContextProvider {
     return chunksCopy;
   }
 
-  private _sortByPreIndexedDocs(
+  private _sortAlphabetically(
     submenuItems: ContextSubmenuItem[],
   ): ContextSubmenuItem[] {
-    // Sort submenuItems such that the objects with titles which don't occur in configs occur first, and alphabetized
+    // Sort submenu items alphabetically by title
     return submenuItems.sort((a, b) => {
-      const aTitleInConfigs = a.metadata?.preIndexed ?? false;
-      const bTitleInConfigs = b.metadata?.preIndexed ?? false;
-
-      // Primary criterion: Items not in configs come first
-      if (!aTitleInConfigs && bTitleInConfigs) {
-        return -1;
-      } else if (aTitleInConfigs && !bTitleInConfigs) {
-        return 1;
-      } else {
-        // Secondary criterion: Alphabetical order when both items are in the same category
-        return a.title.toString().localeCompare(b.title.toString());
-      }
+      return a.title.toString().localeCompare(b.title.toString());
     });
   }
 
@@ -165,46 +153,22 @@ class DocsContextProvider extends BaseContextProvider {
     }
     await docsService.isInitialized;
 
-    // Create map of docs url -> submenu item
-    const submenuItemsMap = new Map<string, ContextSubmenuItem>();
+    // Create an array to hold submenu items
+    const submenuItems: ContextSubmenuItem[] = [];
 
-    // Add custom docs from config
+    // Get all indexed docs from the database
     const docs = (await docsService.listMetadata()) ?? [];
     for (const { startUrl, title, favicon } of docs) {
-      submenuItemsMap.set(startUrl, {
+      submenuItems.push({
         title,
         id: startUrl,
         description: new URL(startUrl).hostname,
         icon: favicon,
       });
     }
 
-    // Add pre-indexed docs if supported
-    const canUsePreindexedDocs = await docsService.canUsePreindexedDocs();
-    if (canUsePreindexedDocs) {
-      for (const { startUrl, title } of Object.values(preIndexedDocs)) {
-        // Skip if overridden in config
-        if (docs.find((d) => d.startUrl === startUrl)) {
-          continue;
-        }
-        submenuItemsMap.set(startUrl, {
-          title,
-          id: startUrl,
-          description: new URL(startUrl).hostname,
-          metadata: {
-            preIndexed: true,
-          },
-        });
-      }
-    }
-
-    // Create array and sort if pre-indexed is supported
-    const submenuItems = Array.from(submenuItemsMap.values());
-    if (canUsePreindexedDocs) {
-      return this._sortByPreIndexedDocs(submenuItems);
-    }
-
-    return submenuItems;
+    // Sort alphabetically
+    return this._sortAlphabetically(submenuItems);
   }
 }
 

core/control-plane/client.ts

@@ -5,6 +5,7 @@ import {
   FQSN,
   FullSlug,
   SecretResult,
+  SecretType,
 } from "@continuedev/config-yaml";
 import fetch, { RequestInit, Response } from "node-fetch";
 
@@ -46,7 +47,14 @@ export class ControlPlaneClient {
   ): Promise<(SecretResult | undefined)[]> {
     const userId = await this.userId;
     if (!userId) {
-      throw new Error("No user id");
+      return fqsns.map((fqsn) => ({
+        found: false,
+        fqsn,
+        secretLocation: {
+          secretName: fqsn.secretName,
+          secretType: SecretType.NotFound,
+        },
+      }));
     }
 
     const resp = await this.request("ide/sync-secrets", {

core/index.d.ts

@@ -236,15 +236,13 @@ export interface SiteIndexingConfig {
   maxDepth?: number;
   faviconUrl?: string;
   useLocalCrawling?: boolean;
-  rootUrl?: string; // Currently only used by preindexed docs
 }
 
 export interface DocsIndexingDetails {
   startUrl: string;
   config: SiteIndexingConfig;
   indexingStatus: IndexingStatus | undefined;
   chunks: Chunk[];
-  isPreIndexedDoc: boolean;
 }
 
 export interface IContextProvider {