added get hoverable range and case insensitive text search

Author: arnavch

.gitignore

@@ -1,3 +1,4 @@
 /target
 /index
-/test_files
+/test_files
+.aider*

Cargo.lock

@@ -79,6 +79,7 @@ zstd = "=0.12.4"
 zstd-safe = "=6.0.6"
 
 zstd-sys = "=2.0.9+zstd.1.5.5"
+serde_json = "1.0.117"
 
 [dev-dependencies]
 pretty_assertions = "1.4.0"

Cargo.toml

@@ -1,12 +1,12 @@
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use tantivy::{schema::Schema, IndexWriter, doc, Term};
 use anyhow::Result;
 use async_trait::async_trait;
 use tokio::fs;
 use tokio::task::spawn_blocking;
 use futures::future::BoxFuture;
 use std::collections::{HashSet, HashMap};
-use crate::{indexes::Indexable, sync_handle::SyncHandle};
+use crate::indexes::Indexable;
 use crate::intelligence::{TreeSitterFile, TSLanguage};
 use crate::symbol::SymbolLocations;
 use crate::schema::build_schema;
@@ -58,20 +58,15 @@ impl File {
 
 #[async_trait]
 impl Indexable for File {
-    async fn index_repository(&self, handle: &SyncHandle, root_path: &Path, writer: &IndexWriter) -> Result<()> {
+    async fn index_repository(&self, root_path: &Path, writer: &IndexWriter) -> Result<()> {
         let existing_docs = load_existing_docs(writer, &self.hash_field, &self.path_field)?;
-        let gitignore = parse_gitignore(root_path).await?;
-
-        // Print all paths in gitignore
-        // println!("Paths in .gitignore:");
-        // for pattern in &patterns {
-        //     println!("{}", pattern);
-        // }
+        let mut gitignore = GlobSetBuilder::new();
 
         traverse_and_index_files(
             root_path, writer, &self.schema, self.path_field, self.content_field,
             self.symbol_locations_field, self.symbols_field, self.line_end_indices_field,
-            self.lang_field, self.hash_field, self.content_insensitive_field, &existing_docs, &gitignore, root_path).await
+            self.lang_field, self.hash_field, self.content_insensitive_field, 
+            &existing_docs, &mut gitignore, root_path).await
     }
 
     fn schema(&self) -> Schema {
@@ -98,21 +93,20 @@ fn load_existing_docs(writer: &IndexWriter, hash_field: &tantivy::schema::Field,
     Ok(existing_docs)
 }
 
-async fn parse_gitignore(repo_path: &Path) -> Result<GlobSet> {
-    let gitignore_path = repo_path.join(".gitignore");
-    let mut builder = GlobSetBuilder::new();
+async fn parse_gitignore(current_path: &Path, builder: &mut GlobSetBuilder) -> Result<()> {
+    let gitignore_path = current_path.join(".gitignore");
 
     if gitignore_path.exists() {
         let contents = tokio::fs::read_to_string(&gitignore_path).await?;
         for line in contents.lines() {
             let trimmed_line = line.trim();
             if !trimmed_line.starts_with('#') && !trimmed_line.is_empty() {
-                let mut absolute_pattern = if trimmed_line.starts_with('/') {
+                let absolute_pattern = if trimmed_line.starts_with('/') {
                     // The pattern is already an absolute path, so we just use it as is
-                    repo_path.join(trimmed_line.trim_start_matches('/'))
+                    current_path.join(trimmed_line.trim_start_matches('/'))
                 } else {
-                    // The pattern is a relative path, so we join it with the repo path
-                    repo_path.join(trimmed_line)
+                    // The pattern is a relative path, so we join it with the current path
+                    current_path.join(trimmed_line)
                 };
                 let pattern = absolute_pattern.to_string_lossy().replace("\\", "/");
                 // println!("Adding to gitignore: {}", pattern);
@@ -121,9 +115,10 @@ async fn parse_gitignore(repo_path: &Path) -> Result<GlobSet> {
         }
     }
 
-    Ok(builder.build()?)
+    Ok(())
 }
 
+
 fn traverse_and_index_files<'a>(
     path: &'a Path,
     writer: &'a IndexWriter,
@@ -137,10 +132,16 @@ fn traverse_and_index_files<'a>(
     hash_field: tantivy::schema::Field,
     content_insensitive_field: tantivy::schema::Field,  // New field
     existing_docs: &'a HashMap<String, String>,
-    gitignore: &'a GlobSet,
+    gitignore: &'a mut GlobSetBuilder,
     root_path: &'a Path,
 ) -> BoxFuture<'a, Result<()>> {
     Box::pin(async move {
+        // Parse .gitignore in the current directory and update the builder
+        parse_gitignore(path, gitignore).await?;
+
+        // Build the GlobSet from the builder
+        let globset = gitignore.build()?;
+
         let mut entries = fs::read_dir(path).await?;
         while let Some(entry) = entries.next_entry().await? {
             let path = entry.path();
@@ -150,7 +151,7 @@ fn traverse_and_index_files<'a>(
             let absolute_path_str = absolute_path.to_string_lossy().replace("\\", "/");
 
             // Skip paths that match .gitignore patterns
-            if gitignore.is_match(&absolute_path_str) {
+            if globset.is_match(&absolute_path_str) {
                 continue;
             }
 
@@ -220,16 +221,11 @@ fn traverse_and_index_files<'a>(
                 let last_char_byte_index = content_str.chars().map(|c| c.len_utf8()).sum::<usize>();
                 line_end_indices.extend_from_slice(&u32::to_le_bytes(last_char_byte_index as u32));
 
-                
-
                 // Convert content to lower case for case-insensitive search
                 let content_insensitive = content_str.to_lowercase();
-                // println!("{}", content_insensitive);
 
                 // println!("{}", absolute_path_str);
 
-                
-
                 let doc = tantivy::doc!(
                     path_field => path_str,
                     content_field => content_str,
@@ -246,4 +242,4 @@ fn traverse_and_index_files<'a>(
         }
         Ok(())
     })
-}
+}

src/file.rs

@@ -3,11 +3,11 @@ use anyhow::{Context, Result};
 use async_trait::async_trait;
 use tantivy::{schema::Schema, Index, IndexReader, IndexWriter};
 use tokio::sync::Mutex;
-use crate::{sync_handle::SyncHandle, file::File};
+use crate::file::File;
 
 #[async_trait]
 pub trait Indexable: Send + Sync {
-    async fn index_repository(&self, handle: &SyncHandle, root_path: &Path, writer: &IndexWriter) -> Result<()>;
+    async fn index_repository(&self, root_path: &Path, writer: &IndexWriter) -> Result<()>;
     fn schema(&self) -> Schema;
 }
 
@@ -18,8 +18,8 @@ pub struct IndexWriteHandle<'a> {
 }
 
 impl<'a> IndexWriteHandle<'a> {
-    pub async fn index(&self, handle: &SyncHandle, root_path: &Path) -> Result<()> {
-        self.source.index_repository(handle, root_path, &self.writer).await
+    pub async fn index(&self, root_path: &Path) -> Result<()> {
+        self.source.index_repository(root_path, &self.writer).await
     }
 
     pub fn commit(&mut self) -> Result<()> {
@@ -105,9 +105,8 @@ impl Indexes {
 
     pub async fn index(&self, root_path: &Path) -> Result<()> {
         let _write_lock = self.write_mutex.lock().await;
-        let handle = SyncHandle::default();
         let mut writer = self.file.write_handle()?;
-        writer.index(&handle, root_path).await?;
+        writer.index( root_path).await?;
         writer.commit()?;
         Ok(())
     }

src/indexes.rs

@@ -6,7 +6,7 @@ use crate::{
     content_document::ContentDocument,
     snippet::{Snipper, Snippet},
     text_range::TextRange,
-    intelligence::{TreeSitterFile, NodeKind},
+    intelligence::NodeKind,
 };
 
 #[derive(Debug, Serialize)]

src/intelligence/code_navigation.rs

@@ -1,12 +1,13 @@
 use anyhow::Result;
 use std::path::Path;
 use retreival::{indexes::Indexes, search::Searcher};
+use serde_json::json;
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    let root_path = Path::new("/Users/arnav/Desktop/r2/retreival");
+    let root_path = Path::new("/Users/arnav/Desktop/devon/Devon");
     // println!("{}", root_path.display());
-    let index_path = Path::new("/Users/arnav/Desktop/r2/retreival/index");
+    let index_path = Path::new("/Users/arnav/Desktop/devon/Devon/index");
 
     let buffer_size_per_thread = 15_000_000;
     let num_threads = 4;
@@ -16,17 +17,23 @@ async fn main() -> Result<()> {
 
     // // // // Create a searcher and perform a search
     let searcher = Searcher::new(&index_path)?;
-    // let result = searcher.token_info("rust", "/Users/arnav/Desktop/r2/retreival/src/main.rs", 18, 19, 27);
-    // println!("{:?}", result);
-    // // // // let results = searcher.load_all_documents("rust");
-    let result = searcher.text_search("indexes")?;
+
+    // let result = searcher.token_info("/Users/arnav/Desktop/r2/retreival/src/main.rs", 14, 18, 25);
+    // match result {
+    //     Ok(token_info) => println!("{}", retreival::search::Searcher::format_token_info(token_info)),
+    //     Err(e) => println!("Error retrieving token info: {}", e),
+    // }
+
+    let result = searcher.text_search("Agent", true)?;
+    println!("{}", retreival::search::Searcher::format_search_results(result));
+
+    // let result = searcher.get_hoverable_ranges("/Users/arnav/Desktop/devon/Devon/devon_agent/tools/edittools.py")?;
+    // println!("{}", json!(retreival::search::Searcher::format_hoverable_ranges(result)).to_string());
 
     // println!("-");
     // // Print out the results
-    for resul in result {
-        println!("{:?}", resul.path);
-        // println!("{:?}", resul.context);
-    }
+    
+
 
 
     Ok(())