diff --git a/core/config/yaml/loadYaml.ts b/core/config/yaml/loadYaml.ts index 1edb486043..ffdae1f3f2 100644 --- a/core/config/yaml/loadYaml.ts +++ b/core/config/yaml/loadYaml.ts @@ -5,6 +5,7 @@ import { ConfigResult, ConfigValidationError, FQSN, + isAssistantUnrolledNonNullable, ModelRole, PlatformClient, RegistryClient, @@ -93,7 +94,15 @@ async function loadConfigYaml( renderSecrets: true, }, )); - const errors = validateConfigYaml(config); + + const errors = isAssistantUnrolledNonNullable(config) + ? validateConfigYaml(config) + : [ + { + fatal: true, + message: "Assistant includes blocks that don't exist", + }, + ]; if (errors?.some((error) => error.fatal)) { return { @@ -123,29 +132,13 @@ async function configYamlToContinueConfig( allowFreeTrial: boolean = true, ): Promise<{ config: ContinueConfig; errors: ConfigValidationError[] }> { const localErrors: ConfigValidationError[] = []; + const continueConfig: ContinueConfig = { slashCommands: [], models: [], tools: [...allTools], mcpServerStatuses: [], - systemMessage: undefined, - experimental: { - modelContextProtocolServers: config.mcpServers?.map((mcpServer) => ({ - transport: { - type: "stdio", - command: mcpServer.command, - args: mcpServer.args ?? [], - env: mcpServer.env, - }, - })), - }, - docs: config.docs?.map((doc) => ({ - title: doc.name, - startUrl: doc.startUrl, - rootUrl: doc.rootUrl, - faviconUrl: doc.faviconUrl, - })), - rules: config.rules, + systemMessage: config.rules?.join("\n"), contextProviders: [], modelsByRole: { chat: [], @@ -165,7 +158,38 @@ async function configYamlToContinueConfig( rerank: null, summarize: null, }, - data: config.data, + }; + + // Right now, if there are any missing packages in the config, then we will just throw an error + if (!isAssistantUnrolledNonNullable(config)) { + return { + config: continueConfig, + errors: [ + { + message: "Found missing blocks in config.yaml", + fatal: true, + }, + ], + }; + } + + continueConfig.data = config.data; + continueConfig.rules = config.rules; + continueConfig.docs = config.docs?.map((doc) => ({ + title: doc.name, + startUrl: doc.startUrl, + rootUrl: doc.rootUrl, + faviconUrl: doc.faviconUrl, + })); + continueConfig.experimental = { + modelContextProtocolServers: config.mcpServers?.map((mcpServer) => ({ + transport: { + type: "stdio", + command: mcpServer.command, + args: mcpServer.args ?? [], + env: mcpServer.env, + }, + })), }; // Prompt files - diff --git a/core/package-lock.json b/core/package-lock.json index 24aeb4f184..daf8696a44 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -13,7 +13,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.777.0", "@aws-sdk/credential-providers": "^3.778.0", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.71", + "@continuedev/config-yaml": "^1.0.77", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.8", "@continuedev/openai-adapters": "^1.0.18", @@ -2977,9 +2977,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.71", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.71.tgz", - "integrity": "sha512-c7KVaYfldaYHf2EqeqE9Oo1son33iTp6MAY+WRPJVs9XWmRWOA6Cn7YDf575FpwX6UsPCT/Eb0JJme8AC7UE6g==", + "version": "1.0.77", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.77.tgz", + "integrity": "sha512-Vke6jp4bENjJc9fO1tavOnbYXRmmSIEzNhYeQ6DzZVl92+8rWwdO6zhbrK0r4FGjHnH0cJgyK/q0zX9/l+6asw==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/core/package.json b/core/package.json index a405bad55c..a007fe6378 100644 --- a/core/package.json +++ b/core/package.json @@ -47,7 +47,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.777.0", "@aws-sdk/credential-providers": "^3.778.0", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.71", + "@continuedev/config-yaml": "^1.0.77", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.8", "@continuedev/openai-adapters": "^1.0.18", diff --git a/gui/package-lock.json b/gui/package-lock.json index 69c571f84d..18a5d148e4 100644 --- a/gui/package-lock.json +++ b/gui/package-lock.json @@ -7,7 +7,7 @@ "name": "gui", "license": "Apache-2.0", "dependencies": { - "@continuedev/config-yaml": "^1.0.71", + "@continuedev/config-yaml": "^1.0.77", "@headlessui/react": "^2.2.0", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", @@ -109,7 +109,7 @@ "@aws-sdk/client-sagemaker-runtime": "^3.777.0", "@aws-sdk/credential-providers": "^3.778.0", "@continuedev/config-types": "^1.0.13", - "@continuedev/config-yaml": "^1.0.71", + "@continuedev/config-yaml": "^1.0.77", "@continuedev/fetch": "^1.0.4", "@continuedev/llm-info": "^1.0.8", "@continuedev/openai-adapters": "^1.0.18", @@ -553,9 +553,9 @@ } }, "node_modules/@continuedev/config-yaml": { - "version": "1.0.71", - "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.71.tgz", - "integrity": "sha512-c7KVaYfldaYHf2EqeqE9Oo1son33iTp6MAY+WRPJVs9XWmRWOA6Cn7YDf575FpwX6UsPCT/Eb0JJme8AC7UE6g==", + "version": "1.0.77", + "resolved": "https://registry.npmjs.org/@continuedev/config-yaml/-/config-yaml-1.0.77.tgz", + "integrity": "sha512-Vke6jp4bENjJc9fO1tavOnbYXRmmSIEzNhYeQ6DzZVl92+8rWwdO6zhbrK0r4FGjHnH0cJgyK/q0zX9/l+6asw==", "dependencies": { "@continuedev/config-types": "^1.0.14", "yaml": "^2.6.1", diff --git a/gui/package.json b/gui/package.json index 2d4ed109b1..98db617a08 100644 --- a/gui/package.json +++ b/gui/package.json @@ -15,7 +15,7 @@ "test:watch": "vitest" }, "dependencies": { - "@continuedev/config-yaml": "^1.0.71", + "@continuedev/config-yaml": "^1.0.77", "@headlessui/react": "^2.2.0", "@heroicons/react": "^2.0.18", "@reduxjs/toolkit": "^2.3.0", diff --git a/gui/src/components/mainInput/Lump/sections/RulesSection.tsx b/gui/src/components/mainInput/Lump/sections/RulesSection.tsx index 62a0738242..cd9e94d919 100644 --- a/gui/src/components/mainInput/Lump/sections/RulesSection.tsx +++ b/gui/src/components/mainInput/Lump/sections/RulesSection.tsx @@ -1,4 +1,4 @@ -import { parseConfigYaml, Rule } from "@continuedev/config-yaml"; +import { parseConfigYaml, type Rule } from "@continuedev/config-yaml"; import { ArrowsPointingOutIcon, PencilIcon } from "@heroicons/react/24/outline"; import { useContext, useMemo } from "react"; import { useSelector } from "react-redux"; diff --git a/packages/config-yaml/package-lock.json b/packages/config-yaml/package-lock.json index bd174fa967..2a8b83224c 100644 --- a/packages/config-yaml/package-lock.json +++ b/packages/config-yaml/package-lock.json @@ -1049,6 +1049,7 @@ "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", "dev": true, + "license": "MIT", "dependencies": { "expect": "^29.0.0", "pretty-format": "^29.0.0" diff --git a/packages/config-yaml/test/index.test.ts b/packages/config-yaml/src/__tests__/index.test.ts similarity index 75% rename from packages/config-yaml/test/index.test.ts rename to packages/config-yaml/src/__tests__/index.test.ts index 71b073f42d..0c8f120d74 100644 --- a/packages/config-yaml/test/index.test.ts +++ b/packages/config-yaml/src/__tests__/index.test.ts @@ -1,18 +1,19 @@ import * as fs from "fs"; -import { decodeSecretLocation, resolveSecretLocationInProxy } from "../dist"; import { + decodeSecretLocation, FQSN, FullSlug, PlatformClient, PlatformSecretStore, Registry, resolveFQSN, + resolveSecretLocationInProxy, SecretLocation, SecretResult, SecretStore, SecretType, unrollAssistant, -} from "../src"; +} from "../index.js"; // Test e2e flows from raw yaml -> unroll -> client render -> resolve secrets on proxy describe("E2E Scenarios", () => { @@ -92,12 +93,28 @@ describe("E2E Scenarios", () => { getContent: async function (fullSlug: FullSlug): Promise { return fs .readFileSync( - `./test/packages/${fullSlug.ownerSlug}/${fullSlug.packageSlug}.yaml`, + `./src/__tests__/packages/${fullSlug.ownerSlug}/${fullSlug.packageSlug}.yaml`, ) .toString(); }, }; + it("should unroll assistant with a single block that doesn't exist", async () => { + const unrolledConfig = await unrollAssistant( + "test-org/assistant-with-non-existing-block", + registry, + { + renderSecrets: true, + platformClient, + orgScopeId: "test-org", + currentUserSlug: "test-user", + onPremProxyUrl: null, + }, + ); + + expect(unrolledConfig.rules?.[0]).toBeNull(); + }); + it("should correctly unroll assistant", async () => { const unrolledConfig = await unrollAssistant( "test-org/assistant", @@ -133,10 +150,10 @@ describe("E2E Scenarios", () => { ); expect(unrolledConfig.rules?.length).toBe(2); - expect(unrolledConfig.docs?.[0].startUrl).toBe( + expect(unrolledConfig.docs?.[0]?.startUrl).toBe( "https://docs.python.org/release/3.13.1", ); - expect(unrolledConfig.docs?.[0].rootUrl).toBe( + expect(unrolledConfig.docs?.[0]?.rootUrl).toBe( "https://docs.python.org/release/3.13.1", ); @@ -176,5 +193,40 @@ describe("E2E Scenarios", () => { expect(geminiSecretValue2).toBe("gemini-api-key"); }); + it("should correctly unroll assistant with injected blocks", async () => { + const unrolledConfig = await unrollAssistant( + "test-org/assistant", + registry, + { + renderSecrets: true, + platformClient, + orgScopeId: "test-org", + currentUserSlug: "test-user", + onPremProxyUrl: null, + // Add an injected block + injectBlocks: [ + { + ownerSlug: "test-org", + packageSlug: "docs", + versionSlug: "latest", + }, + ], + }, + ); + + // The original docs array should have one item + expect(unrolledConfig.docs?.length).toBe(2); // Now 2 with the injected block + + // Check the original doc is still there + expect(unrolledConfig.docs?.[0]?.startUrl).toBe( + "https://docs.python.org/release/3.13.1", + ); + + // Check the injected doc block was added + expect(unrolledConfig.docs?.[1]?.startUrl).toBe( + "https://docs.python.org/release/3.13.1", + ); + }); + it.skip("should prioritize org over user / package secrets", () => {}); }); diff --git a/packages/config-yaml/src/__tests__/packages/test-org/assistant-with-non-existing-block.yaml b/packages/config-yaml/src/__tests__/packages/test-org/assistant-with-non-existing-block.yaml new file mode 100644 index 0000000000..4f01b32b8c --- /dev/null +++ b/packages/config-yaml/src/__tests__/packages/test-org/assistant-with-non-existing-block.yaml @@ -0,0 +1,6 @@ +name: Assistant with non-existing block +version: 0.0.1 +schema: v1 + +rules: + - uses: test-org/non-existing-package diff --git a/packages/config-yaml/test/packages/test-org/assistant.yaml b/packages/config-yaml/src/__tests__/packages/test-org/assistant.yaml similarity index 100% rename from packages/config-yaml/test/packages/test-org/assistant.yaml rename to packages/config-yaml/src/__tests__/packages/test-org/assistant.yaml diff --git a/packages/config-yaml/test/packages/test-org/claude35sonnet.yaml b/packages/config-yaml/src/__tests__/packages/test-org/claude35sonnet.yaml similarity index 100% rename from packages/config-yaml/test/packages/test-org/claude35sonnet.yaml rename to packages/config-yaml/src/__tests__/packages/test-org/claude35sonnet.yaml diff --git a/packages/config-yaml/test/packages/test-org/docs.yaml b/packages/config-yaml/src/__tests__/packages/test-org/docs.yaml similarity index 100% rename from packages/config-yaml/test/packages/test-org/docs.yaml rename to packages/config-yaml/src/__tests__/packages/test-org/docs.yaml diff --git a/packages/config-yaml/test/packages/test-org/gemini.yaml b/packages/config-yaml/src/__tests__/packages/test-org/gemini.yaml similarity index 100% rename from packages/config-yaml/test/packages/test-org/gemini.yaml rename to packages/config-yaml/src/__tests__/packages/test-org/gemini.yaml diff --git a/packages/config-yaml/test/packages/test-org/rules.yaml b/packages/config-yaml/src/__tests__/packages/test-org/rules.yaml similarity index 100% rename from packages/config-yaml/test/packages/test-org/rules.yaml rename to packages/config-yaml/src/__tests__/packages/test-org/rules.yaml diff --git a/packages/config-yaml/src/load/clientRender.ts b/packages/config-yaml/src/load/clientRender.ts index 585fc24eee..ffc62f0fc4 100644 --- a/packages/config-yaml/src/load/clientRender.ts +++ b/packages/config-yaml/src/load/clientRender.ts @@ -120,16 +120,17 @@ export function useProxyForUnrenderedSecrets( if (config.models) { for (let i = 0; i < config.models.length; i++) { const apiKeyLocation = getUnrenderedSecretLocation( - config.models[i].apiKey, + config.models[i]?.apiKey, ); if (apiKeyLocation) { config.models[i] = { ...config.models[i], + name: config.models[i]?.name ?? "", provider: "continue-proxy", model: getContinueProxyModelName( packageSlug, - config.models[i].provider, - config.models[i].model, + config.models[i]?.provider ?? "", + config.models[i]?.model ?? "", ), apiKeyLocation: encodeSecretLocation(apiKeyLocation), orgScopeId, diff --git a/packages/config-yaml/src/load/unroll.ts b/packages/config-yaml/src/load/unroll.ts index 3d56b0251e..1e818c4ca9 100644 --- a/packages/config-yaml/src/load/unroll.ts +++ b/packages/config-yaml/src/load/unroll.ts @@ -20,6 +20,7 @@ import { Rule, } from "../schemas/index.js"; import { useProxyForUnrenderedSecrets } from "./clientRender.js"; +import { getBlockType } from "./getBlockType.js"; export function parseConfigYaml(configYaml: string): ConfigYaml { try { @@ -178,11 +179,18 @@ async function extractRenderedSecretsMap( return map; } -export interface DoNotRenderSecretsUnrollAssistantOptions { +export interface BaseUnrollAssistantOptions { + renderSecrets: boolean; + injectBlocks?: FullSlug[]; +} + +export interface DoNotRenderSecretsUnrollAssistantOptions + extends BaseUnrollAssistantOptions { renderSecrets: false; } -export interface RenderSecretsUnrollAssistantOptions { +export interface RenderSecretsUnrollAssistantOptions + extends BaseUnrollAssistantOptions { renderSecrets: true; orgScopeId: string | null; currentUserSlug: string; @@ -239,7 +247,11 @@ export async function unrollAssistantFromContent( let parsedYaml = parseConfigYaml(rawYaml); // Unroll blocks and convert their secrets to FQSNs - const unrolledAssistant = await unrollBlocks(parsedYaml, registry); + const unrolledAssistant = await unrollBlocks( + parsedYaml, + registry, + options.injectBlocks, + ); // Back to a string so we can fill in template variables const rawUnrolledYaml = YAML.stringify(unrolledAssistant); @@ -277,6 +289,7 @@ export async function unrollAssistantFromContent( export async function unrollBlocks( assistant: ConfigYaml, registry: Registry, + injectBlocks: FullSlug[] | undefined, ): Promise { const unrolledAssistant: AssistantUnrolled = { name: assistant.name, @@ -296,16 +309,23 @@ export async function unrollBlocks( for (const unrolledBlock of assistant[section]) { // "uses/with" block if ("uses" in unrolledBlock) { - const blockConfigYaml = await resolveBlock( - decodeFullSlug(unrolledBlock.uses), - unrolledBlock.with, - registry, - ); - const block = blockConfigYaml[section]?.[0]; - if (block) { - sectionBlocks.push( - mergeOverrides(block, unrolledBlock.override ?? {}), + try { + const blockConfigYaml = await resolveBlock( + decodeFullSlug(unrolledBlock.uses), + unrolledBlock.with, + registry, + ); + const block = blockConfigYaml[section]?.[0]; + if (block) { + sectionBlocks.push( + mergeOverrides(block, unrolledBlock.override ?? {}), + ); + } + } catch (err) { + console.error( + `Failed to unroll block ${unrolledBlock.uses}: ${(err as Error).message}`, ); + sectionBlocks.push(null); } } else { // Normal block @@ -319,19 +339,26 @@ export async function unrollBlocks( // Rules are a bit different because they can be strings, so hanlde separately if (assistant.rules) { - const rules: Rule[] = []; + const rules: (Rule | null)[] = []; for (const rule of assistant.rules) { if (typeof rule === "string" || !("uses" in rule)) { rules.push(rule); } else if ("uses" in rule) { - const blockConfigYaml = await resolveBlock( - decodeFullSlug(rule.uses), - rule.with, - registry, - ); - const block = blockConfigYaml.rules?.[0]; - if (block) { - rules.push(block); + try { + const blockConfigYaml = await resolveBlock( + decodeFullSlug(rule.uses), + rule.with, + registry, + ); + const block = blockConfigYaml.rules?.[0]; + if (block) { + rules.push(block); + } + } catch (err) { + console.error( + `Failed to unroll block ${rule.uses}: ${(err as Error).message}`, + ); + rules.push(null); } } } @@ -339,6 +366,33 @@ export async function unrollBlocks( unrolledAssistant.rules = rules; } + // Add injected blocks + for (const injectBlock of injectBlocks ?? []) { + try { + const blockConfigYaml = await registry.getContent(injectBlock); + const parsedBlock = parseConfigYaml(blockConfigYaml); + const blockType = getBlockType(parsedBlock); + const resolvedBlock = await resolveBlock( + injectBlock, + undefined, + registry, + ); + + if (blockType) { + if (!unrolledAssistant[blockType]) { + unrolledAssistant[blockType] = []; + } + unrolledAssistant[blockType]?.push( + ...(resolvedBlock[blockType] as any), + ); + } + } catch (err) { + console.error( + `Failed to unroll block ${injectBlock}: ${(err as Error).message}`, + ); + } + } + return unrolledAssistant; } diff --git a/packages/config-yaml/src/schemas/index.ts b/packages/config-yaml/src/schemas/index.ts index ae12c99a7b..e941ec0254 100644 --- a/packages/config-yaml/src/schemas/index.ts +++ b/packages/config-yaml/src/schemas/index.ts @@ -88,6 +88,18 @@ export const configYamlSchema = baseConfigYamlSchema.extend({ export type ConfigYaml = z.infer; export const assistantUnrolledSchema = baseConfigYamlSchema.extend({ + models: z.array(modelSchema.nullable()).optional(), + context: z.array(contextSchema.nullable()).optional(), + data: z.array(dataSchema.nullable()).optional(), + mcpServers: z.array(mcpServerSchema.nullable()).optional(), + rules: z.array(ruleSchema.nullable()).optional(), + prompts: z.array(promptSchema.nullable()).optional(), + docs: z.array(docSchema.nullable()).optional(), +}); + +export type AssistantUnrolled = z.infer; + +export const assistantUnrolledSchemaNonNullable = baseConfigYamlSchema.extend({ models: z.array(modelSchema).optional(), context: z.array(contextSchema).optional(), data: z.array(dataSchema).optional(), @@ -97,7 +109,20 @@ export const assistantUnrolledSchema = baseConfigYamlSchema.extend({ docs: z.array(docSchema).optional(), }); -export type AssistantUnrolled = z.infer; +export type AssistantUnrolledNonNullable = z.infer< + typeof assistantUnrolledSchemaNonNullable +>; + +export const isAssistantUnrolledNonNullable = ( + a: AssistantUnrolled, +): a is AssistantUnrolledNonNullable => + (!a.models || a.models.every((m) => m !== null)) && + (!a.context || a.context.every((c) => c !== null)) && + (!a.data || a.data.every((d) => d !== null)) && + (!a.mcpServers || a.mcpServers.every((s) => s !== null)) && + (!a.rules || a.rules.every((r) => r !== null)) && + (!a.prompts || a.prompts.every((p) => p !== null)) && + (!a.docs || a.docs.every((d) => d !== null)); export const blockSchema = baseConfigYamlSchema.and( z.union([