Feature/v2/autodoc local

.gitignore

@@ -65,4 +65,54 @@ lerna-debug.log
 tsdoc_cache/
 tsdoc_comments/
 
-**/elizaDb/**
+**/elizaDb/**# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data
+
+packages/autodoc/text_objects/*/*
+agent/data/db.sqlite

packages/autodoc/objectStore.ts

@@ -0,0 +1,67 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+// Define the dictionary to track occurrences
+const textDict: { [key: string]: number } = {};
+
+// Directory for storing text objects
+const OBJECTS_DIR = './text_objects';
+
+// Ensure the objects directory exists
+if (!fs.existsSync(OBJECTS_DIR)) {
+    fs.mkdirSync(OBJECTS_DIR);
+}
+
+// Function to generate SHA-1 hash
+function generateHash(content: string): string {
+    return crypto.createHash('sha1')
+        .update(content)
+        .digest('hex');
+}
+
+// Function to write content to object store
+function writeToObjectStore(content: string): string {
+    const hash = generateHash(content);
+    const dir = path.join(OBJECTS_DIR, hash.substring(0, 2));
+    const filename = path.join(dir, hash.substring(2));
+
+    //console.log(`DEBUG: ${hash}`);
+
+    // Create directory if it doesn't exist
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+
+    // Only write if file doesn't exist
+    if (!fs.existsSync(filename)) {
+        fs.writeFileSync(filename, content);
+        console.log(`Stored new text with hash: ${hash}`);
+    } else  {
+        //console.log(`Text with hash ${hash} already exists, skipping...`);
+   }
+
+    return hash;
+}
+
+export function processText(finalPrompt: string): void {
+    const trimmedText = finalPrompt.trim();
+
+    if (!trimmedText) {
+        console.log("Empty text, skipping...");
+        return;
+    }
+
+    if (textDict[trimmedText]) {
+        // Text exists, increment count and report if it becomes a duplicate
+        textDict[trimmedText]++;
+        //if (textDict[trimmedText] === 2) {
+            console.log("Duplicate text found:", trimmedText);
+        //}
+    } else {
+        // New text, add to dictionary and store
+        textDict[trimmedText] = 1;
+        writeToObjectStore(trimmedText);
+    }
+}
+

packages/autodoc/src/AIService/AIService.ts

@@ -4,6 +4,7 @@ import type { Configuration } from "../Configuration.js";
 import { TypeScriptParser } from "../TypeScriptParser.js";
 import { CodeFormatter } from "./utils/CodeFormatter.js";
 import { DocumentOrganizer } from "./utils/DocumentOrganizer.js";
+import { processText } from "./objectStore.js";
 
 dotenv.config();
 
@@ -25,78 +26,97 @@ export class AIService {
 	 * @param {Configuration} configuration - The configuration instance to be used
 	 * @throws {Error} If OPENAI_API_KEY environment variable is not set
 	 */
-	constructor(private configuration: Configuration) {
-		if (!process.env.OPENAI_API_KEY) {
-			throw new Error("OPENAI_API_KEY is not set");
-		}
-		this.chatModel = new ChatOpenAI({ apiKey: process.env.OPENAI_API_KEY });
-		this.chatModelFAQ = new ChatOpenAI({
-			apiKey: process.env.OPENAI_API_KEY,
-			model: "gpt-4o",
-		});
-		this.codeFormatter = new CodeFormatter();
-	}
+    constructor(private configuration: Configuration) {
 
-	/**
-	 * Generates a comment based on the specified prompt by invoking the chat model.
-	 * @param {string} prompt - The prompt for which to generate a comment
-	 * @returns {Promise<string>} The generated comment
-	 */
-	public async generateComment(prompt: string, isFAQ = false): Promise<string> {
-		try {
-			// First try with generous limit
-			let finalPrompt = prompt;
-			if (!isFAQ) {
-				finalPrompt = this.codeFormatter.truncateCodeBlock(prompt, 8000);
-			}
 
-			console.log(
-				`Generating comment for prompt of length: ${finalPrompt.length}`,
-			);
+	    if (!process.env.OPENAI_API_KEY) {
+		throw new Error("OPENAI_API_KEY is not set");
+	    }
+
+	    let defaultCreds = { apiKey: process.env.OPENAI_API_KEY , configuration:{}}
+	    if (process.env.OPENAI_API_BASE) {
+		//console.log("OPENAI_API_BASE",process.env.OPENAI_API_BASE);
+		defaultCreds.configuration = { "basePath" : process.env.OPENAI_API_BASE }
+	    }
 
-			try {
-				let response;
-				if (isFAQ) {
-					response = await this.chatModelFAQ.invoke(finalPrompt);
-				} else {
-					response = await this.chatModel.invoke(finalPrompt);
-				}
+	    //console.log("DEFAULT_CREDS",defaultCreds);
+	    this.chatModel = new ChatOpenAI(defaultCreds)
+	    this.chatModelFAQ = new ChatOpenAI({ ...defaultCreds, ...{model: "gpt-4o"}});
+	    this.codeFormatter = new CodeFormatter();
+	}
+
+	private async _generateComment(finalPrompt: string, isFAQ = false): Promise<string> {
+		if (!process.env.NOAI) {
+		    try {
+			let response;
+
+			if (isFAQ) {
+			    response = await this.chatModelFAQ.invoke(finalPrompt);
+			} else {
+			    response = await this.chatModel.invoke(finalPrompt);
+			}
+			return response.content as string;		    
+
+		    } catch (error) {
+			if (
+			    error instanceof Error &&
+				error.message.includes("maximum context length")
+			) {
+			    console.warn(
+				"Token limit exceeded, attempting with further truncation...",
+			    );
+			    // Try with more aggressive truncation
+			    finalPrompt = this.codeFormatter.truncateCodeBlock(finalPrompt, 4000);
+			    try {
+				const response = await this.chatModel.invoke(finalPrompt);
 				return response.content as string;
-			} catch (error) {
+			    } catch (retryError) {
 				if (
-					error instanceof Error &&
-					error.message.includes("maximum context length")
+				    retryError instanceof Error &&
+					retryError.message.includes("maximum context length")
 				) {
-					console.warn(
-						"Token limit exceeded, attempting with further truncation...",
-					);
-					// Try with more aggressive truncation
-					finalPrompt = this.codeFormatter.truncateCodeBlock(prompt, 4000);
-					try {
-						const response = await this.chatModel.invoke(finalPrompt);
-						return response.content as string;
-					} catch (retryError) {
-						if (
-							retryError instanceof Error &&
-							retryError.message.includes("maximum context length")
-						) {
-							console.warn(
-								"Still exceeding token limit, using minimal context...",
-							);
-							// Final attempt with minimal context
-							finalPrompt = this.codeFormatter.truncateCodeBlock(prompt, 2000);
-							const response = await this.chatModel.invoke(finalPrompt);
-							return response.content as string;
-						}
-						throw retryError;
-					}
+				    console.warn(
+					"Still exceeding token limit, using minimal context...",
+				    );
+				    // Final attempt with minimal context
+				    finalPrompt = this.codeFormatter.truncateCodeBlock(finalPrompt, 2000);
+				    const response = await this.chatModel.invoke(finalPrompt);
+				    return response.content as string;
 				}
-				throw error;
+				throw retryError;
+			    }
 			}
-		} catch (error) {
-			this.handleAPIError(error as Error);
-			return "";
+			throw error;
+		    }
 		}
+		else {
+		    //console.log("SKIP");
+		    return '{ "title": "no ai", "body": "body" }'
+		}					    
+
+	}
+	/**
+	 * Generates a comment based on the specified prompt by invoking the chat model.
+	 * @param {string} prompt - The prompt for which to generate a comment
+	 * @returns {Promise<string>} The generated comment
+	 */
+	public async generateComment(prompt: string, isFAQ = false): Promise<string> {
+
+		// First try with generous limit
+		let finalPrompt = prompt;
+		if (!isFAQ) {
+		    finalPrompt = this.codeFormatter.truncateCodeBlock(prompt, 8000);
+		}
+
+		console.log(
+		    `Generating comment for prompt of length: ${finalPrompt.length}`,
+		);
+
+		let result= await this._generateComment(finalPrompt, isFAQ);
+		processText(finalPrompt, result);
+
+		return result
+
 	}
 
 	/**
@@ -111,3 +131,6 @@ export class AIService {
 		throw error;
 	}
 }
+
+
+