chore(api): Changed the search api to return raw source (instead of base64 encoding) (#356)

This PR alters the behaviour of the search api (and all apis that depend on it) to return raw source code instead of a base64 encoding. Reasoning: we are decoding it on the client in multiple different places, so it would be beneficial to decode it in a single spot.

**Note**: This is a **breaking change** to the API surface. However, since the API surface is still unofficial/unsupported, I will roll this as a patch version change. See #101

Author: brendan-kellam

CHANGELOG.md

@@ -7,10 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+<!-- @NOTE: this release includes a API change that affects the MCP package (@sourcebot/mcp). On release, bump the MCP package's version and delete this message. -->
+
 ### Fixed
 - Delete account join request when redeeming an invite. [#352](https://github.com/sourcebot-dev/sourcebot/pull/352)
 - Fix issue where a repository would not be included in a search context if the context was created before the repository. [#354](https://github.com/sourcebot-dev/sourcebot/pull/354)
 
+### Changed
+- Changed search api (and all apis that depend on it) to return raw source code instead of base64 encoded string. ([356](https://github.com/sourcebot-dev/sourcebot/pull/356)).
+
+
 ## [4.3.0] - 2025-06-11
 
 ### Added

packages/mcp/CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+- Updated API client to match the latest Sourcebot release. [#356](https://github.com/sourcebot-dev/sourcebot/pull/356)
+
 ## [1.0.2] - 2025-05-28
 
 ### Changed

packages/mcp/src/index.ts

@@ -8,7 +8,7 @@ import { z } from 'zod';
 import { listRepos, search, getFileSource } from './client.js';
 import { env, numberSchema } from './env.js';
 import { TextContent } from './types.js';
-import { base64Decode, isServiceError } from './utils.js';
+import { isServiceError } from './utils.js';
 
 // Create MCP server
 const server = new McpServer({
@@ -114,8 +114,7 @@ server.tool(
 
             if (includeCodeSnippets) {
                 const snippets = file.chunks.map(chunk => {
-                    const content = base64Decode(chunk.content);
-                    return `\`\`\`\n${content}\n\`\`\``
+                    return `\`\`\`\n${chunk.content}\n\`\`\``
                 }).join('\n');
                 text += `\n\n${snippets}`;
             }
@@ -201,7 +200,7 @@ server.tool(
 
         const content: TextContent[] = [{
             type: "text",
-            text: `file: ${fileName}\nrepository: ${repoId}\nlanguage: ${response.language}\nsource:\n${base64Decode(response.source)}`,
+            text: `file: ${fileName}\nrepository: ${repoId}\nlanguage: ${response.language}\nsource:\n${response.source}`,
         }]
 
         return {

packages/mcp/src/utils.ts

@@ -1,10 +1,5 @@
 import { ServiceError } from "./types.js";
 
-// From https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem
-export const base64Decode = (base64: string): string => {
-    const binString = atob(base64);
-    return Buffer.from(Uint8Array.from(binString, (m) => m.codePointAt(0)!).buffer).toString();
-}
 
 export const isServiceError = (data: unknown): data is ServiceError => {
     return typeof data === 'object' &&

packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { base64Decode, getCodeHostInfoForRepo, unwrapServiceError } from "@/lib/utils";
+import { getCodeHostInfoForRepo, unwrapServiceError } from "@/lib/utils";
 import { useBrowseParams } from "@/app/[domain]/browse/hooks/useBrowseParams";
 import { useQuery } from "@tanstack/react-query";
 import { getFileSource } from "@/features/search/fileSourceApi";
@@ -88,7 +88,7 @@ export const CodePreviewPanel = () => {
             </div>
             <Separator />
             <PureCodePreviewPanel
-                source={base64Decode(fileSourceResponse.source)}
+                source={fileSourceResponse.source}
                 language={fileSourceResponse.language}
                 repoName={repoName}
                 path={path}

packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx

@@ -7,7 +7,6 @@ import { useDomain } from "@/hooks/useDomain";
 import { SymbolIcon } from "@radix-ui/react-icons";
 import { SetStateAction, Dispatch, useMemo } from "react";
 import { getFileSource } from "@/features/search/fileSourceApi";
-import { base64Decode } from "@/lib/utils";
 import { unwrapServiceError } from "@/lib/utils";
 
 interface CodePreviewPanelProps {
@@ -41,10 +40,8 @@ export const CodePreviewPanel = ({
             }, domain)
         ),
         select: (data) => {
-            const decodedSource = base64Decode(data.source);
-
             return {
-                content: decodedSource,
+                content: data.source,
                 filepath: previewedFile.fileName.text,
                 matches: previewedFile.chunks,
                 link: previewedFile.webUrl,

packages/web/src/app/[domain]/search/components/searchResultsPanel/fileMatch.tsx

@@ -1,8 +1,7 @@
 'use client';
 
-import { useCallback, useMemo } from "react";
+import { useCallback } from "react";
 import { SearchResultFile, SearchResultChunk } from "@/features/search/types";
-import { base64Decode } from "@/lib/utils";
 import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter";
 
 
@@ -17,17 +16,12 @@ export const FileMatch = ({
     file,
     onOpen: _onOpen,
 }: FileMatchProps) => {
-
-    const content = useMemo(() => {
-        return base64Decode(match.content);
-    }, [match.content]);
-
     const onOpen = useCallback((isCtrlKeyPressed: boolean) => {
         const startLineNumber = match.contentStart.lineNumber;
-        const endLineNumber = content.trimEnd().split('\n').length + startLineNumber - 1;
+        const endLineNumber = match.content.trimEnd().split('\n').length + startLineNumber - 1;
 
         _onOpen(startLineNumber, endLineNumber, isCtrlKeyPressed);
-    }, [content, match.contentStart.lineNumber, _onOpen]);
+    }, [match.content, match.contentStart.lineNumber, _onOpen]);
 
     // If it's just the title, don't show a code preview
     if (match.matchRanges.length === 0) {
@@ -57,7 +51,7 @@ export const FileMatch = ({
                 lineNumbersOffset={match.contentStart.lineNumber}
                 renderWhitespace={true}
             >
-                {content}
+                {match.content}
             </LightweightCodeHighlighter>
         </div>
     );

packages/web/src/ee/features/codeNav/components/exploreMenu/referenceList.tsx

@@ -5,7 +5,6 @@ import { PathHeader } from "@/app/[domain]/components/pathHeader";
 import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter";
 import { FindRelatedSymbolsResponse } from "@/features/codeNav/types";
 import { RepositoryInfo, SourceRange } from "@/features/search/types";
-import { base64Decode } from "@/lib/utils";
 import { useMemo, useRef } from "react";
 import useCaptureEvent from "@/hooks/useCaptureEvent";
 import { useVirtualizer } from "@tanstack/react-virtual";
@@ -155,10 +154,6 @@ const ReferenceListItem = ({
     onClick,
     onMouseEnter,
 }: ReferenceListItemProps) => {
-    const decodedLineContent = useMemo(() => {
-        return base64Decode(lineContent);
-    }, [lineContent]);
-
     const highlightRanges = useMemo(() => [range], [range]);
 
     return (
@@ -174,7 +169,7 @@ const ReferenceListItem = ({
                 lineNumbersOffset={range.start.lineNumber}
                 renderWhitespace={false}
             >
-                {decodedLineContent}
+                {lineContent}
             </LightweightCodeHighlighter>
         </div>
     )

packages/web/src/ee/features/codeNav/components/symbolHoverPopup/symbolDefinitionPreview.tsx

@@ -3,7 +3,6 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
 import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter";
 import { useMemo } from "react";
 import { SourceRange } from "@/features/search/types";
-import { base64Decode } from "@/lib/utils";
 
 interface SymbolDefinitionPreviewProps {
     symbolDefinition: {
@@ -21,10 +20,6 @@ export const SymbolDefinitionPreview = ({
     const { lineContent, language, range } = symbolDefinition;
     const highlightRanges = useMemo(() => [range], [range]);
 
-    const decodedLineContent = useMemo(() => {
-        return base64Decode(lineContent);
-    }, [lineContent]);
-
     return (
         <div className="flex flex-col gap-2 mb-2">
             <Tooltip
@@ -55,7 +50,7 @@ export const SymbolDefinitionPreview = ({
                 lineNumbersOffset={range.start.lineNumber}
                 renderWhitespace={false}
             >
-                {decodedLineContent}
+                {lineContent}
             </LightweightCodeHighlighter>
         </div>
     )

packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts

@@ -1,7 +1,6 @@
 import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/review-agent/types";
 import { getFileSource } from "@/features/search/fileSourceApi";
 import { fileSourceResponseSchema } from "@/features/search/schemas";
-import { base64Decode } from "@/lib/utils";
 import { isServiceError } from "@/lib/utils";
 import { env } from "@/env.mjs";
 import { createLogger } from "@sourcebot/logger";
@@ -24,7 +23,7 @@ export const fetchFileContent = async (pr_payload: sourcebot_pr_payload, filenam
     }
 
     const fileSourceResponse = fileSourceResponseSchema.parse(response);
-    const fileContent = base64Decode(fileSourceResponse.source);
+    const fileContent = fileSourceResponse.source;
 
     const fileContentContext: sourcebot_context = {
         type: "file_content",

packages/web/src/features/search/searchApi.ts

@@ -10,6 +10,7 @@ import { SearchRequest, SearchResponse, SourceRange } from "./types";
 import { OrgRole, Repo } from "@sourcebot/db";
 import * as Sentry from "@sentry/nextjs";
 import { sew, withAuth, withOrgMembership } from "@/actions";
+import { base64Decode } from "@sourcebot/shared";
 
 // List of supported query prefixes in zoekt.
 // @see : https://github.com/sourcebot-dev/zoekt/blob/main/query/parse.go#L417
@@ -264,7 +265,7 @@ export const search = async ({ query, matches, contextLines, whole }: SearchRequ
                             .filter((chunk) => !chunk.FileName) // Filter out filename chunks.
                             .map((chunk) => {
                                 return {
-                                    content: chunk.Content,
+                                    content: base64Decode(chunk.Content),
                                     matchRanges: chunk.Ranges.map((range) => ({
                                         start: {
                                             byteOffset: range.Start.ByteOffset,
@@ -295,7 +296,7 @@ export const search = async ({ query, matches, contextLines, whole }: SearchRequ
                                 }
                             }),
                         branches: file.Branches,
-                        content: file.Content,
+                        content: file.Content ? base64Decode(file.Content) : undefined,
                     }
                 }).filter((file) => file !== undefined) ?? [];
 

packages/web/src/lib/utils.ts

@@ -301,12 +301,6 @@ export const isServiceError = (data: unknown): data is ServiceError => {
         'message' in data;
 }
 
-// From https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem
-export const base64Decode = (base64: string): string => {
-    const binString = atob(base64);
-    return Buffer.from(Uint8Array.from(binString, (m) => m.codePointAt(0)!).buffer).toString();
-}
-
 // @see: https://stackoverflow.com/a/65959350/23221295
 export const isDefined = <T>(arg: T | null | undefined): arg is T extends null | undefined ? never : T => {
     return arg !== null && arg !== undefined;