Skip to content

Instruction-based tools #5809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 17 commits into
base: main
Choose a base branch
from
Draft

Instruction-based tools #5809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6a6cbd1
tools input
RomneyDa May 21, 2025
57b2880
non-native tool calling working
RomneyDa May 22, 2025
92c3415
merge main
RomneyDa May 22, 2025
5324e95
working non-native tools
RomneyDa May 23, 2025
c948ae1
cleanup non-native tool calls
RomneyDa May 23, 2025
2a58814
non-native tools system prompt iteration and fixes
RomneyDa May 23, 2025
f067dde
non native tool fixes
RomneyDa May 23, 2025
b4a52e2
non native tools system prompt iteration
RomneyDa May 23, 2025
028964e
merge main
RomneyDa May 24, 2025
017bb24
non-native tools use tools logic
RomneyDa May 25, 2025
f20282b
framework for system message instructions per tool definition
RomneyDa May 26, 2025
ae6c8e2
remove builtin prefix from tool names
RomneyDa May 27, 2025
73addb6
update builtin tool names throughout, move instruction tools to core
RomneyDa May 27, 2025
7fe6c2d
custom system messages for read, diff, edit, create"
RomneyDa May 27, 2025
cecb941
specific definitions for read current file, run terminal command, sea…
RomneyDa May 27, 2025
d1bae2c
merge main
RomneyDa May 27, 2025
ce28d58
util for generating openai tool call id
RomneyDa May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/config/load.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,7 @@ async function intermediateToFinalConfig({
return { config: continueConfig, errors };
}

function llmToSerializedModelDescription(llm: ILLM): ModelDescription {
export function llmToSerializedModelDescription(llm: ILLM): ModelDescription {
return {
provider: llm.providerName,
underlyingProviderName: llm.underlyingProviderName,
Expand Down
1 change: 0 additions & 1 deletion core/config/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1122,7 +1122,6 @@ declare global {
* This is needed to crawl a large number of documentation sites that are dynamically rendered.
*/
useChromiumForDocsCrawling?: boolean;
useTools?: boolean;
modelContextProtocolServers?: MCPOptions[];
}

Expand Down
1 change: 1 addition & 0 deletions core/index.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,7 @@ export interface Tool {
uri?: string;
faviconUrl?: string;
group: string;
systemMessageDescription?: string;
}

interface ToolChoice {
Expand Down
21 changes: 1 addition & 20 deletions core/llm/autodetect.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
import {
ChatMessage,
ModelCapability,
ModelDescription,
TemplateType,
} from "../index.js";
import { ChatMessage, ModelCapability, TemplateType } from "../index.js";

import {
anthropicTemplateMessages,
Expand Down Expand Up @@ -41,7 +36,6 @@ import {
xWinCoderEditPrompt,
zephyrEditPrompt,
} from "./templates/edit.js";
import { PROVIDER_TOOL_SUPPORT } from "./toolSupport.js";

const PROVIDER_HANDLES_TEMPLATING: string[] = [
"lmstudio",
Expand Down Expand Up @@ -103,17 +97,6 @@ const MODEL_SUPPORTS_IMAGES: string[] = [
"granite-vision",
];

function modelSupportsTools(modelDescription: ModelDescription) {
if (modelDescription.capabilities?.tools !== undefined) {
return modelDescription.capabilities.tools;
}
const providerSupport = PROVIDER_TOOL_SUPPORT[modelDescription.provider];
if (!providerSupport) {
return false;
}
return providerSupport(modelDescription.model) ?? false;
}

function modelSupportsImages(
provider: string,
model: string,
Expand Down Expand Up @@ -389,6 +372,4 @@ export {
autodetectTemplateType,
llmCanGenerateInParallel,
modelSupportsImages,
modelSupportsTools
};

43 changes: 24 additions & 19 deletions core/llm/llms/Bedrock.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
} from "@aws-sdk/client-bedrock-runtime";
import { fromNodeProviderChain } from "@aws-sdk/credential-providers";

import { llmToSerializedModelDescription } from "../../config/load.js";
import {
ChatMessage,
Chunk,
Expand All @@ -17,7 +18,7 @@ import {
} from "../../index.js";
import { renderChatMessage, stripImages } from "../../util/messageContent.js";
import { BaseLLM } from "../index.js";
import { PROVIDER_TOOL_SUPPORT } from "../toolSupport.js";
import { modelSupportsNativeTools } from "../toolSupport.js";
import { getSecureID } from "../utils/getSecureID.js";

interface ModelConfig {
Expand Down Expand Up @@ -292,9 +293,12 @@ class Bedrock extends BaseLLM {
const convertedMessages = this._convertMessages(messages);

const shouldCacheSystemMessage =
!!systemMessage && this.cacheBehavior?.cacheSystemMessage || this.completionOptions.promptCaching;
(!!systemMessage && this.cacheBehavior?.cacheSystemMessage) ||
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using !!options.tools is redundant. Consider using just options.tools since the && operator will short-circuit if options.tools is falsy.

this.completionOptions.promptCaching;
const enablePromptCaching =
shouldCacheSystemMessage || this.cacheBehavior?.cacheConversation || this.completionOptions.promptCaching;
shouldCacheSystemMessage ||
this.cacheBehavior?.cacheConversation ||
this.completionOptions.promptCaching;
const shouldCacheToolsConfig = this.completionOptions.promptCaching;

// Add header for prompt caching
Expand All @@ -305,22 +309,23 @@ class Bedrock extends BaseLLM {
};
}

const supportsTools =
(this.capabilities?.tools || PROVIDER_TOOL_SUPPORT.bedrock?.(options.model)) ?? false;

let toolConfig = supportsTools && options.tools
? {
tools: options.tools.map((tool) => ({
toolSpec: {
name: tool.function.name,
description: tool.function.description,
inputSchema: {
json: tool.function.parameters,
},
},
})),
} as ToolConfiguration
: undefined;
const modelDesc = llmToSerializedModelDescription(this);
const supportsTools = modelSupportsNativeTools(modelDesc);

let toolConfig =
supportsTools && !!options.tools
? ({
tools: options.tools.map((tool) => ({
toolSpec: {
name: tool.function.name,
description: tool.function.description,
inputSchema: {
json: tool.function.parameters,
},
},
})),
} as ToolConfiguration)
: undefined;

if (toolConfig?.tools && shouldCacheToolsConfig) {
toolConfig.tools.push({ cachePoint: { type: "default" } });
Expand Down
32 changes: 16 additions & 16 deletions core/llm/toolSupport.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// core/llm/toolSupport.test.ts
import { PROVIDER_TOOL_SUPPORT } from "./toolSupport";
import { NATIVE_TOOL_SUPPORT } from "./toolSupport";

describe("PROVIDER_TOOL_SUPPORT", () => {
describe("NATIVE_TOOL_SUPPORT", () => {
describe("continue-proxy", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["continue-proxy"];
const supportsFn = NATIVE_TOOL_SUPPORT["continue-proxy"];

it("should return true for Claude 3.5 models", () => {
expect(
Expand Down Expand Up @@ -61,7 +61,7 @@ describe("PROVIDER_TOOL_SUPPORT", () => {
});

describe("anthropic", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["anthropic"];
const supportsFn = NATIVE_TOOL_SUPPORT["anthropic"];

it("should return true for Claude 3.5 models", () => {
expect(supportsFn("claude-3-5-sonnet")).toBe(true);
Expand All @@ -85,7 +85,7 @@ describe("PROVIDER_TOOL_SUPPORT", () => {
});

describe("openai", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["openai"];
const supportsFn = NATIVE_TOOL_SUPPORT["openai"];

it("should return true for GPT-4 models", () => {
expect(supportsFn("gpt-4")).toBe(true);
Expand All @@ -110,7 +110,7 @@ describe("PROVIDER_TOOL_SUPPORT", () => {
});

describe("gemini", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["gemini"];
const supportsFn = NATIVE_TOOL_SUPPORT["gemini"];

it("should return true for all Gemini models", () => {
expect(supportsFn("gemini-pro")).toBe(true);
Expand All @@ -130,7 +130,7 @@ describe("PROVIDER_TOOL_SUPPORT", () => {
});

describe("bedrock", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["bedrock"];
const supportsFn = NATIVE_TOOL_SUPPORT["bedrock"];

it("should return true for Claude 3.5 Sonnet models", () => {
expect(supportsFn("anthropic.claude-3-5-sonnet-20240620-v1:0")).toBe(
Expand Down Expand Up @@ -180,7 +180,7 @@ describe("PROVIDER_TOOL_SUPPORT", () => {
});

describe("mistral", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["mistral"];
const supportsFn = NATIVE_TOOL_SUPPORT["mistral"];

it("should return true for supported models", () => {
expect(supportsFn("mistral-large-latest")).toBe(true);
Expand Down Expand Up @@ -212,7 +212,7 @@ describe("PROVIDER_TOOL_SUPPORT", () => {
});

describe("ollama", () => {
const supportsFn = PROVIDER_TOOL_SUPPORT["ollama"];
const supportsFn = NATIVE_TOOL_SUPPORT["ollama"];

it("should return true for supported models", () => {
expect(supportsFn("llama3.1")).toBe(true);
Expand Down Expand Up @@ -256,17 +256,17 @@ describe("PROVIDER_TOOL_SUPPORT", () => {

describe("edge cases", () => {
it("should handle empty model names", () => {
expect(PROVIDER_TOOL_SUPPORT["continue-proxy"]("")).toBe(false);
expect(PROVIDER_TOOL_SUPPORT["anthropic"]("")).toBe(false);
expect(PROVIDER_TOOL_SUPPORT["openai"]("")).toBe(false);
expect(PROVIDER_TOOL_SUPPORT["gemini"]("")).toBe(false);
expect(PROVIDER_TOOL_SUPPORT["bedrock"]("")).toBe(false);
expect(PROVIDER_TOOL_SUPPORT["ollama"]("")).toBe(false);
expect(NATIVE_TOOL_SUPPORT["continue-proxy"]("")).toBe(false);
expect(NATIVE_TOOL_SUPPORT["anthropic"]("")).toBe(false);
expect(NATIVE_TOOL_SUPPORT["openai"]("")).toBe(false);
expect(NATIVE_TOOL_SUPPORT["gemini"]("")).toBe(false);
expect(NATIVE_TOOL_SUPPORT["bedrock"]("")).toBe(false);
expect(NATIVE_TOOL_SUPPORT["ollama"]("")).toBe(false);
});

it("should handle non-existent provider", () => {
// @ts-ignore - Testing runtime behavior with invalid provider
expect(PROVIDER_TOOL_SUPPORT["non-existent"]).toBe(undefined);
expect(NATIVE_TOOL_SUPPORT["non-existent"]).toBe(undefined);
});
});
});
Loading
Loading