Add search engine feature

openhands/agenthub/codeact_agent/codeact_agent.py

@@ -70,6 +70,7 @@ def __init__(
             codeact_enable_browsing=self.config.codeact_enable_browsing,
             codeact_enable_jupyter=self.config.codeact_enable_jupyter,
             codeact_enable_llm_editor=self.config.codeact_enable_llm_editor,
+            codeact_enable_search_engine=self.config.enable_search_engine,
             llm=self.llm,
         )
         logger.debug(

openhands/agenthub/codeact_agent/function_calling.py

@@ -15,6 +15,7 @@
     FinishTool,
     IPythonTool,
     LLMBasedFileEditTool,
+    SearchEngineTool,
     ThinkTool,
     WebReadTool,
     create_cmd_run_tool,
@@ -36,6 +37,7 @@
     FileReadAction,
     IPythonRunCellAction,
     MessageAction,
+    SearchAction,
 )
 from openhands.events.event import FileEditSource, FileReadSource
 from openhands.events.tool import ToolCallMetadata
@@ -191,6 +193,15 @@ def response_to_actions(response: ModelResponse) -> list[Action]:
                         f'Missing required argument "url" in tool call {tool_call.function.name}'
                     )
                 action = BrowseURLAction(url=arguments['url'])
+            # ================================================
+            # SearchEngineTool (search the web using text queries)
+            # ================================================
+            elif tool_call.function.name == SearchEngineTool['function']['name']:
+                if 'query' not in arguments:
+                    raise FunctionCallNotExistsError(
+                        f'Missing required argument "query" in tool call {tool_call.function.name}'
+                    )
+                action = SearchAction(query=arguments['query'])
             else:
                 raise FunctionCallNotExistsError(
                     f'Tool {tool_call.function.name} is not registered. (arguments: {arguments}). Please check the tool name and retry with an existing tool.'
@@ -223,6 +234,7 @@ def get_tools(
     codeact_enable_browsing: bool = False,
     codeact_enable_llm_editor: bool = False,
     codeact_enable_jupyter: bool = False,
+    codeact_enable_search_engine: bool = False,
     llm: LLM | None = None,
 ) -> list[ChatCompletionToolParam]:
     SIMPLIFIED_TOOL_DESCRIPTION_LLM_SUBSTRS = ['gpt-', 'o3', 'o1']
@@ -239,6 +251,8 @@ def get_tools(
         ThinkTool,
         FinishTool,
     ]
+    if codeact_enable_search_engine:
+        tools.append(SearchEngineTool)
     if codeact_enable_browsing:
         tools.append(WebReadTool)
         tools.append(BrowserTool)

openhands/agenthub/codeact_agent/tools/__init__.py

@@ -3,6 +3,7 @@
 from .finish import FinishTool
 from .ipython import IPythonTool
 from .llm_based_edit import LLMBasedFileEditTool
+from .search_engine import SearchEngineTool
 from .str_replace_editor import create_str_replace_editor_tool
 from .think import ThinkTool
 from .web_read import WebReadTool
@@ -13,6 +14,7 @@
     'FinishTool',
     'IPythonTool',
     'LLMBasedFileEditTool',
+    'SearchEngineTool',
     'create_str_replace_editor_tool',
     'WebReadTool',
     'ThinkTool',

openhands/agenthub/codeact_agent/tools/search_engine.py

@@ -0,0 +1,24 @@
+from litellm import ChatCompletionToolParam, ChatCompletionToolParamFunctionChunk
+
+_SEARCH_ENGINE_DESCRIPTION = """Execute a web search query (similar to Google search).
+
+NOTE: When you need to search for information online, please use the `search_engine` tool rather than the `browser` or `web_read` tools. The `search_engine` tool connects directly to a search engine, which will help avoid CAPTCHA challenges that would otherwise block your access.
+"""
+
+SearchEngineTool = ChatCompletionToolParam(
+    type='function',
+    function=ChatCompletionToolParamFunctionChunk(
+        name='search_engine',
+        description=_SEARCH_ENGINE_DESCRIPTION,
+        parameters={
+            'type': 'object',
+            'properties': {
+                'query': {
+                    'type': 'string',
+                    'description': 'The web search query (must be a non-empty string).',
+                },
+            },
+            'required': ['query'],
+        },
+    ),
+)

openhands/core/config/agent_config.py

@@ -2,7 +2,10 @@
 
 from pydantic import BaseModel, Field, ValidationError
 
-from openhands.core.config.condenser_config import CondenserConfig, NoOpCondenserConfig
+from openhands.core.config.condenser_config import (
+    CondenserConfig,
+    NoOpCondenserConfig,
+)
 from openhands.core.logger import openhands_logger as logger
 
 
@@ -30,6 +33,7 @@ class AgentConfig(BaseModel):
     disabled_microagents: list[str] = Field(default_factory=list)
     enable_history_truncation: bool = Field(default=True)
     enable_som_visual_browsing: bool = Field(default=False)
+    enable_search_engine: bool = Field(default=False)
     condenser: CondenserConfig = Field(default_factory=NoOpCondenserConfig)
 
     model_config = {'extra': 'forbid'}

openhands/core/schema/action.py

@@ -82,6 +82,9 @@ class ActionTypeSchema(BaseModel):
     SEND_PR: str = Field(default='send_pr')
     """Send a PR to github."""
 
+    SEARCH: str = Field(default='search')
+    """Queries a search engine."""
+
     RECALL: str = Field(default='recall')
     """Retrieves content from a user workspace, microagent, or other source."""
 

openhands/core/schema/observation.py

@@ -49,6 +49,9 @@ class ObservationTypeSchema(BaseModel):
     CONDENSE: str = Field(default='condense')
     """Result of a condensation operation."""
 
+    SEARCH: str = Field(default='search')
+    """Result of querying a search engine."""
+
     RECALL: str = Field(default='recall')
     """Result of a recall operation. This can be the workspace context, a microagent, or other types of information."""
 

openhands/events/action/__init__.py

@@ -17,6 +17,7 @@
     FileWriteAction,
 )
 from openhands.events.action.message import MessageAction
+from openhands.events.action.search_engine import SearchAction
 
 __all__ = [
     'Action',
@@ -36,5 +37,6 @@
     'MessageAction',
     'ActionConfirmationStatus',
     'AgentThinkAction',
+    'SearchAction',
     'RecallAction',
 ]

openhands/events/action/search_engine.py

@@ -0,0 +1,24 @@
+from dataclasses import dataclass
+from typing import ClassVar
+
+from openhands.core.schema import ActionType
+from openhands.events.action.action import Action
+
+
+@dataclass
+class SearchAction(Action):
+    query: str
+    thought: str = ''
+    action: str = ActionType.SEARCH
+    runnable: ClassVar[bool] = True
+
+    @property
+    def message(self) -> str:
+        return f'I am querying the search engine to search for {self.query}'
+
+    def __str__(self) -> str:
+        ret = '**SearchAction**\n'
+        if self.thought:
+            ret += f'THOUGHT: {self.thought}\n'
+        ret += f'QUERY: {self.query}'
+        return ret

openhands/events/observation/__init__.py

@@ -5,6 +5,7 @@
     AgentThinkObservation,
     RecallObservation,
 )
+from openhands.events.observation.search_engine import SearchEngineObservation
 from openhands.events.observation.browse import BrowserOutputObservation
 from openhands.events.observation.commands import (
     CmdOutputMetadata,
@@ -42,6 +43,7 @@
     'SuccessObservation',
     'UserRejectObservation',
     'AgentCondensationObservation',
+    'SearchEngineObservation',
     'RecallObservation',
     'RecallType',
 ]

openhands/events/observation/search_engine.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+
+from openhands.core.schema import ObservationType
+from openhands.events.observation.observation import Observation
+
+
+@dataclass
+class SearchEngineObservation(Observation):
+    query: str
+    observation: str = ObservationType.SEARCH
+
+    @property
+    def message(self) -> str:
+        return f'Searched for: {self.query}'
+
+    def __str__(self) -> str:
+        ret = (
+            '**SearchEngineObservation**\n'
+            f'Query: {self.query}\n'
+            f'Search Results: {self.content}\n'
+        )
+        return ret

openhands/events/serialization/action.py

@@ -22,6 +22,7 @@
     FileWriteAction,
 )
 from openhands.events.action.message import MessageAction
+from openhands.events.action.search_engine import SearchAction
 
 actions = (
     NullAction,
@@ -39,6 +40,7 @@
     RecallAction,
     ChangeAgentStateAction,
     MessageAction,
+    SearchAction,
 )
 
 ACTION_TYPE_TO_CLASS = {action_class.action: action_class for action_class in actions}  # type: ignore[attr-defined]

openhands/memory/conversation_memory.py

@@ -27,6 +27,7 @@
     FileEditObservation,
     FileReadObservation,
     IPythonRunCellObservation,
+    SearchEngineObservation,
     UserRejectObservation,
 )
 from openhands.events.observation.agent import (
@@ -385,6 +386,9 @@ def _process_observation(
         elif isinstance(obs, AgentCondensationObservation):
             text = truncate_content(obs.content, max_message_chars)
             message = Message(role='user', content=[TextContent(text=text)])
+        elif isinstance(obs, SearchEngineObservation):
+            text = truncate_content(obs.content, max_message_chars)
+            message = Message(role='user', content=[TextContent(text=text)])
         elif (
             isinstance(obs, RecallObservation)
             and self.agent_config.enable_prompt_extensions

openhands/runtime/action_execution_server.py

@@ -41,6 +41,7 @@
     FileReadAction,
     FileWriteAction,
     IPythonRunCellAction,
+    SearchAction,
 )
 from openhands.events.event import FileEditSource, FileReadSource
 from openhands.events.observation import (
@@ -56,6 +57,7 @@
 from openhands.runtime.browser import browse
 from openhands.runtime.browser.browser_env import BrowserEnv
 from openhands.runtime.plugins import ALL_PLUGINS, JupyterPlugin, Plugin, VSCodePlugin
+from openhands.runtime.search_engine.brave_search import search
 from openhands.runtime.utils.bash import BashSession
 from openhands.runtime.utils.files import insert_lines, read_lines
 from openhands.runtime.utils.memory_monitor import MemoryMonitor
@@ -163,7 +165,6 @@ def __init__(
         self.start_time = time.time()
         self.last_execution_time = self.start_time
         self._initialized = False
-
         self.max_memory_gb: int | None = None
         if _override_max_memory_gb := os.environ.get('RUNTIME_MAX_MEMORY_GB', None):
             self.max_memory_gb = int(_override_max_memory_gb)
@@ -464,6 +465,10 @@ async def browse(self, action: BrowseURLAction) -> Observation:
     async def browse_interactive(self, action: BrowseInteractiveAction) -> Observation:
         return await browse(action, self.browser)
 
+    async def search(self, action: SearchAction) -> Observation:
+        obs = await call_sync_from_async(search, action)
+        return obs
+
     def close(self):
         self.memory_monitor.stop_monitoring()
         if self.bash_session is not None:

openhands/runtime/impl/action_execution/action_execution_client.py

@@ -24,6 +24,7 @@
     FileReadAction,
     FileWriteAction,
     IPythonRunCellAction,
+    SearchAction,
 )
 from openhands.events.action.action import Action
 from openhands.events.action.files import FileEditSource
@@ -297,6 +298,9 @@ def browse(self, action: BrowseURLAction) -> Observation:
     def browse_interactive(self, action: BrowseInteractiveAction) -> Observation:
         return self.send_action_for_execution(action)
 
+    def search(self, action: SearchAction) -> Observation:
+        return self.send_action_for_execution(action)
+
     def close(self) -> None:
         # Make sure we don't close the session multiple times
         # Can happen in evaluation

openhands/runtime/search_engine/__init__.py

@@ -0,0 +1,3 @@
+from openhands.runtime.search_engine.brave_search import search
+
+__all__ = ['search']