Merge pull request #5833 from continuedev/nate/vllm-no-fim

Nate/vllm-no-fim

Author: sestinj

core/llm/llms/Vllm.ts

@@ -12,6 +12,10 @@ class Vllm extends OpenAI {
     }
   }
 
+  supportsFim(): boolean {
+    return false;
+  }
+
   private _setupCompletionOptions() {
     this.fetch(this._getEndpoint("models"), {
       method: "GET",

core/llm/llms/stubs/ContinueProxy.ts

@@ -89,6 +89,10 @@ class ContinueProxy extends OpenAI {
   }
 
   supportsFim(): boolean {
+    const { provider } = parseProxyModelName(this.model);
+    if (provider === "vllm") {
+      return false;
+    }
     return true;
   }
 

core/llm/llms/test/supportsFim.test.ts

@@ -0,0 +1,55 @@
+import { jest } from "@jest/globals";
+import Anthropic from "../Anthropic.js";
+import Deepseek from "../Deepseek.js";
+import FunctionNetwork from "../FunctionNetwork.js";
+import Mistral from "../Mistral.js";
+import OpenAI from "../OpenAI.js";
+import ContinueProxy from "../stubs/ContinueProxy.js";
+import Vllm from "../Vllm.js";
+
+// Mock the parseProxyModelName function
+const mockParseProxyModelName = jest.fn();
+jest.mock("@continuedev/config-yaml", () => ({
+  parseProxyModelName: mockParseProxyModelName,
+  decodeSecretLocation: jest.fn(),
+  SecretType: { NotFound: "not-found" },
+}));
+
+// Test cases: [LLM class, model, expected supportsFim result, description]
+const testCases: [any, string, boolean, string][] = [
+  [ContinueProxy, "owner/package/vllm/some-model", false, "vllm provider"],
+  [ContinueProxy, "owner/package/openai/gpt-4", true, "openai provider"],
+  [
+    ContinueProxy,
+    "owner/package/anthropic/claude-3",
+    true,
+    "anthropic provider",
+  ],
+  [ContinueProxy, "owner/package/cohere/command-r", true, "cohere provider"],
+  [
+    ContinueProxy,
+    "owner/package/unknown-provider/some-model",
+    true,
+    "unknown provider",
+  ],
+  [ContinueProxy, "owner/package/groq/llama-model", true, "groq provider"],
+  [Vllm, "any-model", false, "Vllm"],
+  [Anthropic, "claude-3-5-sonnet-latest", false, "Anthropic"],
+  [FunctionNetwork, "any-model", false, "FunctionNetwork"],
+  [OpenAI, "codestral", false, "OpenAI"],
+  [Mistral, "codestral", true, "Mistral"],
+  [Deepseek, "deepseek-chat", true, "Deepseek"],
+];
+
+testCases.forEach(([LLMClass, model, expectedResult, description]) => {
+  test(`supportsFim returns ${expectedResult} for ${description}`, () => {
+    const llm = new LLMClass({
+      model,
+      apiKey: "test-key",
+    });
+
+    const result = llm.supportsFim();
+
+    expect(result).toBe(expectedResult);
+  });
+});

core/llm/toolSupport.ts

@@ -18,7 +18,7 @@ export const PROVIDER_TOOL_SUPPORT: Record<string, (model: string) => boolean> =
         "claude-3.5",
         "claude-3-7",
         "claude-3.7",
-        "claude-4",
+        "claude-sonnet-4",
         "gpt-4",
         "o3",
         "gemini",
@@ -31,7 +31,7 @@ export const PROVIDER_TOOL_SUPPORT: Record<string, (model: string) => boolean> =
           "claude-3.5",
           "claude-3-7",
           "claude-3.7",
-          "claude-4",
+          "claude-sonnet-4",
         ].some((part) => model.toLowerCase().startsWith(part))
       ) {
         return true;
@@ -91,7 +91,7 @@ export const PROVIDER_TOOL_SUPPORT: Record<string, (model: string) => boolean> =
           "claude-3.5",
           "claude-3-7",
           "claude-3.7",
-          "claude-4",
+          "claude-sonnet-4",
         ].some((part) => model.toLowerCase().includes(part))
       ) {
         return true;
@@ -104,6 +104,7 @@ export const PROVIDER_TOOL_SUPPORT: Record<string, (model: string) => boolean> =
       return (
         !model.toLowerCase().includes("mamba") &&
         [
+          "devstral",
           "codestral",
           "mistral-large",
           "mistral-small",

extensions/vscode/package.json

@@ -2,7 +2,7 @@
   "name": "continue",
   "icon": "media/icon.png",
   "author": "Continue Dev, Inc",
-  "version": "1.1.39",
+  "version": "1.1.40",
   "repository": {
     "type": "git",
     "url": "https://github.com/continuedev/continue"