diff --git a/CHANGELOG.md b/CHANGELOG.md index be9cddb..a59f418 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `TLMChatCompletion` module, providing support for trust scoring with OpenAI ChatCompletion objects - Added a VPC compatible version of `TLMChatCompletion` +### Changed + +- Revised tools prompt in `chat.py` + ### Fixed -- Bug fix for formatting system prompt after user messages +- Bug fix in `chat.py` for formatting system prompt after user messages +- Bug fix in `chat.py` for empty tool list still using tools prompt +- Bug fix in `chat.py` for handling empty strings args ## [1.1.9] - 2025-06-17 diff --git a/src/cleanlab_tlm/utils/chat.py b/src/cleanlab_tlm/utils/chat.py index c97a716..ed8e66d 100644 --- a/src/cleanlab_tlm/utils/chat.py +++ b/src/cleanlab_tlm/utils/chat.py @@ -15,6 +15,7 @@ _SYSTEM_PREFIX = "System: " _USER_PREFIX = "User: " _ASSISTANT_PREFIX = "Assistant: " +_TOOL_PREFIX = "Tool: " # Define role constants _SYSTEM_ROLE: Literal["system"] = "system" @@ -41,11 +42,12 @@ # Define tool-related message prefixes _TOOL_DEFINITIONS_PREFIX = ( - "You are a function calling AI model. You are provided with function signatures within " + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " f"{_TOOLS_TAG_START} {_TOOLS_TAG_END} XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " f"{_TOOL_RESPONSE_TAG_START} {_TOOL_RESPONSE_TAG_END} XML tags.\n\n" f"{_TOOLS_TAG_START}\n" ) @@ -231,7 +233,7 @@ def _form_prompt_responses_api( last_system_idx = _find_index_after_first_system_block(messages) # Insert tool definitions and instructions after system messages if needed - if tools is not None: + if tools is not None and len(tools) > 0: messages.insert( last_system_idx + 1, { @@ -272,11 +274,12 @@ def _form_prompt_responses_api( # Format function call as JSON within XML tags, now including call_id function_call = { "name": msg["name"], - "arguments": json.loads(msg["arguments"]), + "arguments": json.loads(msg["arguments"]) if msg["arguments"] else {}, "call_id": call_id, } output += f"{_TOOL_CALL_TAG_START}\n{json.dumps(function_call, indent=2)}\n{_TOOL_CALL_TAG_END}\n\n" elif msg["type"] == _FUNCTION_CALL_OUTPUT_TYPE: + output += _TOOL_PREFIX call_id = msg.get("call_id", "") name = function_names.get(call_id, "function") # Format function response as JSON within XML tags @@ -319,7 +322,7 @@ def _form_prompt_chat_completions_api( # Find the index after the first consecutive block of system messages last_system_idx = _find_index_after_first_system_block(cast(list[dict[str, Any]], messages)) - if tools is not None: + if tools is not None and len(tools) > 0: messages.insert( last_system_idx + 1, { @@ -329,7 +332,7 @@ def _form_prompt_chat_completions_api( ) # Only return content directly if there's a single user message AND no tools - if len(messages) == 1 and messages[0].get("role") == _USER_ROLE and tools is None: + if len(messages) == 1 and messages[0].get("role") == _USER_ROLE and (tools is None or len(tools) == 0): return output + str(messages[0]["content"]) # Warn if the last message is an assistant message with tool calls @@ -359,12 +362,15 @@ def _form_prompt_chat_completions_api( # Format function call as JSON within XML tags, now including call_id function_call = { "name": tool_call["function"]["name"], - "arguments": json.loads(tool_call["function"]["arguments"]), + "arguments": json.loads(tool_call["function"]["arguments"]) + if tool_call["function"]["arguments"] + else {}, "call_id": call_id, } output += f"{_TOOL_CALL_TAG_START}\n{json.dumps(function_call, indent=2)}\n{_TOOL_CALL_TAG_END}\n\n" elif msg["role"] == _TOOL_ROLE: # Handle tool responses + output += _TOOL_PREFIX call_id = msg["tool_call_id"] name = function_names.get(call_id, "function") # Format function response as JSON within XML tags diff --git a/tests/test_chat.py b/tests/test_chat.py index 8760b28..99cbe51 100644 --- a/tests/test_chat.py +++ b/tests/test_chat.py @@ -79,10 +79,12 @@ def test_form_prompt_string_with_tools_chat_completions() -> None: } ] expected = ( - "System: You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "System: You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","function":{"name":"search","description":"Search the web for information","parameters":' @@ -131,10 +133,12 @@ def test_form_prompt_string_with_tools_responses() -> None: } ] expected = ( - "System: You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "System: You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","name":"fetch_user_flight_information","description":"Fetch all tickets for the user along with corresponding flight information and seat assignments.\\n\\n' @@ -198,6 +202,7 @@ def test_form_prompt_string_with_tool_calls_chat_completions() -> None: ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "get_weather",\n' @@ -237,6 +242,7 @@ def test_form_prompt_string_with_tool_calls_responses() -> None: ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "get_weather",\n' @@ -287,6 +293,7 @@ def test_form_prompt_string_with_tool_calls_two_user_messages_chat_completions() ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "get_weather",\n' @@ -330,6 +337,7 @@ def test_form_prompt_string_with_tool_calls_two_user_messages_responses() -> Non ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "get_weather",\n' @@ -369,10 +377,12 @@ def test_form_prompt_string_with_tools_and_system_chat_completions() -> None: ] expected = ( "System: You are ACME Support, the official AI assistant for ACME Corporation. Your role is to provide exceptional customer service and technical support. You are knowledgeable about all ACME products and services, and you maintain a warm, professional, and solution-oriented approach. You can search our knowledge base to provide accurate and up-to-date information about our products, policies, and support procedures.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","function":{"name":"search","description":"Search the web for information","parameters":' @@ -417,10 +427,12 @@ def test_form_prompt_string_with_tools_and_system_responses() -> None: ] expected = ( "System: You are ACME Support, the official AI assistant for ACME Corporation. Your role is to provide exceptional customer service and technical support. You are knowledgeable about all ACME products and services, and you maintain a warm, professional, and solution-oriented approach. You can search our knowledge base to provide accurate and up-to-date information about our products, policies, and support procedures.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","name":"search","description":"Search the web for information","parameters":' @@ -531,10 +543,12 @@ def test_form_prompt_string_warns_on_tool_call_last_responses() -> None: } ] responses_tools_expected = ( - "System: You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "System: You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","name":"fetch_user_flight_information","description":"Fetch flight information","parameters":' @@ -596,6 +610,7 @@ def test_form_prompt_string_assistant_content_before_tool_calls_chat_completions ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "search_knowledge_base",\n' @@ -637,6 +652,7 @@ def test_form_prompt_string_assistant_content_before_tool_calls_responses() -> N ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "search_knowledge_base",\n' @@ -678,14 +694,17 @@ def test_form_prompt_string_with_instructions_and_tools_responses() -> None: ] expected = ( "System: Always be concise and direct in your responses.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","name":"search","description":"Search the web for information","parameters":' - '{"type":"object","properties":{"query":{"type":"string","description":"The search query"}},"required":["query"]},"strict":true}\n' + '{"type":"object","properties":{"query":{"type":"string","description":"The search query"}},"required":["query"]},' + '"strict":true}\n' "\n\n" "For each function call return a JSON object, with the following pydantic model json schema:\n" "{'name': , 'arguments': }\n" @@ -733,6 +752,7 @@ def test_form_prompt_string_with_instructions_and_tool_calls_responses() -> None ' "call_id": "call_123"\n' "}\n" "\n\n" + "Tool: " "\n" "{\n" ' "name": "get_weather",\n' @@ -800,10 +820,12 @@ def test_form_prompt_string_with_developer_role_and_tools() -> None: ] expected = ( "System: Always be concise and direct in your responses.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","name":"search","description":"Search the web for information","parameters":' @@ -847,10 +869,12 @@ def test_form_prompt_string_with_instructions_developer_role_and_tools() -> None expected = ( "System: This system prompt appears first.\n\n" "Always be concise and direct in your responses.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " " XML tags.\n\n" "\n" '{"type":"function","name":"search","description":"Search the web for information","parameters":' @@ -980,8 +1004,9 @@ def test_form_prompt_responses_api_does_not_mutate_messages(use_tools: bool) -> ) -def test_form_prompt_string_with_tools_after_first_system_block_chat_completions() -> None: - """Test that tools are inserted after the first consecutive block of system messages.""" +@pytest.mark.parametrize("use_responses", [False, True]) +def test_form_prompt_string_with_tools_after_first_system_block(use_responses: bool) -> None: + """Test that tools are inserted after the first consecutive block of system messages in both formats.""" messages = [ {"role": "system", "content": "First system message."}, {"role": "system", "content": "Second system message."}, @@ -990,10 +1015,12 @@ def test_form_prompt_string_with_tools_after_first_system_block_chat_completions {"role": "system", "content": "Third system message later."}, {"role": "user", "content": "Tell me more."}, ] - tools = [ - { - "type": "function", - "function": { + + if use_responses: + # Responses format includes strict field + tools = [ + { + "type": "function", "name": "search", "description": "Search the web for information", "parameters": { @@ -1001,96 +1028,186 @@ def test_form_prompt_string_with_tools_after_first_system_block_chat_completions "properties": {"query": {"type": "string", "description": "The search query"}}, "required": ["query"], }, - }, - } - ] + "strict": True, + } + ] + expected = ( + "System: First system message.\n\n" + "Second system message.\n\n" + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " + " XML tags.\n\n" + "\n" + '{"type":"function","name":"search","description":"Search the web for information","parameters":' + '{"type":"object","properties":{"query":{"type":"string","description":"The search query"}},"required":["query"]},' + '"strict":true}\n' + "\n\n" + "For each function call return a JSON object, with the following pydantic model json schema:\n" + "{'name': , 'arguments': }\n" + "Each function call should be enclosed within XML tags.\n" + "Example:\n" + "\n" + "{'name': , 'arguments': }\n" + "\n\n" + "Note: Your past messages will include a call_id in the XML tags. " + "However, do not generate your own call_id when making a function call.\n\n" + "User: What can you do?\n\n" + "Assistant: I can help you.\n\n" + "System: Third system message later.\n\n" + "User: Tell me more.\n\n" + "Assistant:" + ) + else: + # Chat completions format uses nested function structure + tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search the web for information", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string", "description": "The search query"}}, + "required": ["query"], + }, + }, + } + ] + expected = ( + "System: First system message.\n\n" + "Second system message.\n\n" + "You are an AI Assistant that can call provided tools (a.k.a. functions). " + "The set of available tools is provided to you as function signatures within " + " XML tags. " + "You may call one or more of these functions to assist with the user query. If the provided functions are not helpful/relevant, " + "then just respond in natural conversational language. Don't make assumptions about what values to plug " + "into functions. After you choose to call a function, you will be provided with the function's results within " + " XML tags.\n\n" + "\n" + '{"type":"function","function":{"name":"search","description":"Search the web for information","parameters":' + '{"type":"object","properties":{"query":{"type":"string","description":"The search query"}},"required":["query"]}}}\n' + "\n\n" + "For each function call return a JSON object, with the following pydantic model json schema:\n" + "{'name': , 'arguments': }\n" + "Each function call should be enclosed within XML tags.\n" + "Example:\n" + "\n" + "{'name': , 'arguments': }\n" + "\n\n" + "Note: Your past messages will include a call_id in the XML tags. " + "However, do not generate your own call_id when making a function call.\n\n" + "User: What can you do?\n\n" + "Assistant: I can help you.\n\n" + "System: Third system message later.\n\n" + "User: Tell me more.\n\n" + "Assistant:" + ) - result = form_prompt_string(messages, tools) + result = form_prompt_string(messages, tools, use_responses=use_responses) + assert result == expected - expected = ( - "System: First system message.\n\n" - "Second system message.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " - " XML tags.\n\n" - "\n" - '{"type":"function","function":{"name":"search","description":"Search the web for information","parameters":' - '{"type":"object","properties":{"query":{"type":"string","description":"The search query"}},"required":["query"]}}}\n' - "\n\n" - "For each function call return a JSON object, with the following pydantic model json schema:\n" - "{'name': , 'arguments': }\n" - "Each function call should be enclosed within XML tags.\n" - "Example:\n" - "\n" - "{'name': , 'arguments': }\n" - "\n\n" - "Note: Your past messages will include a call_id in the XML tags. " - "However, do not generate your own call_id when making a function call.\n\n" - "User: What can you do?\n\n" - "Assistant: I can help you.\n\n" - "System: Third system message later.\n\n" - "User: Tell me more.\n\n" - "Assistant:" - ) - assert result == expected +@pytest.mark.parametrize("use_responses", [False, True]) +def test_form_prompt_string_with_empty_tools(use_responses: bool) -> None: + """Test that empty tools list is treated the same as None in both formats.""" + messages = [{"role": "user", "content": "Just one message."}] + + # Test with None + result_none = form_prompt_string(messages, tools=None, use_responses=use_responses) + # Test with empty list + result_empty = form_prompt_string(messages, tools=[], use_responses=use_responses) -def test_form_prompt_string_with_tools_after_first_system_block_responses() -> None: - """Test that tools are inserted after the first consecutive block of system messages in responses format.""" + # They should be identical + assert result_none == result_empty == "Just one message." + + +@pytest.mark.parametrize("use_responses", [False, True]) +def test_form_prompt_string_with_empty_tools_multiple_messages(use_responses: bool) -> None: + """Test empty tools list with multiple messages in both formats.""" messages = [ - {"role": "system", "content": "First system message."}, - {"role": "system", "content": "Second system message."}, - {"role": "user", "content": "What can you do?"}, - {"role": "assistant", "content": "I can help you."}, - {"role": "system", "content": "Third system message later."}, - {"role": "user", "content": "Tell me more."}, - ] - tools = [ - { - "type": "function", - "name": "search", - "description": "Search the web for information", - "parameters": { - "type": "object", - "properties": {"query": {"type": "string", "description": "The search query"}}, - "required": ["query"], - }, - "strict": True, - } + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi there!"}, + {"role": "user", "content": "How are you?"}, ] - result = form_prompt_string(messages, tools) + # Test with None + result_none = form_prompt_string(messages, tools=None, use_responses=use_responses) + + # Test with empty list + result_empty = form_prompt_string(messages, tools=[], use_responses=use_responses) + + # They should be identical + expected = "User: Hello!\n\n" "Assistant: Hi there!\n\n" "User: How are you?\n\n" "Assistant:" + assert result_none == result_empty == expected + + +@pytest.mark.parametrize("use_responses", [False, True]) +def test_form_prompt_string_with_empty_arguments(use_responses: bool) -> None: + """Test formatting with tool calls having empty arguments string in both formats.""" + if use_responses: + # Responses format + messages: list[dict[str, Any]] = [ + {"role": "user", "content": "Execute the action"}, + { + "type": "function_call", + "name": "execute_action", + "arguments": "", + "call_id": "call_123", + }, + { + "type": "function_call_output", + "call_id": "call_123", + "output": "Action completed successfully", + }, + ] + else: + # Chat completions format + messages = [ + {"role": "user", "content": "Execute the action"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "type": "function", + "id": "call_123", + "function": { + "name": "execute_action", + "arguments": "", + }, + } + ], + }, + { + "role": "tool", + "name": "execute_action", + "tool_call_id": "call_123", + "content": "Action completed successfully", + }, + ] expected = ( - "System: First system message.\n\n" - "Second system message.\n\n" - "You are a function calling AI model. You are provided with function signatures within XML tags. " - "You may call one or more functions to assist with the user query. If available tools are not relevant in assisting " - "with user query, just respond in natural conversational language. Don't make assumptions about what values to plug " - "into functions. After calling & executing the functions, you will be provided with function results within " - " XML tags.\n\n" - "\n" - '{"type":"function","name":"search","description":"Search the web for information","parameters":' - '{"type":"object","properties":{"query":{"type":"string","description":"The search query"}},"required":["query"]},' - '"strict":true}\n' - "\n\n" - "For each function call return a JSON object, with the following pydantic model json schema:\n" - "{'name': , 'arguments': }\n" - "Each function call should be enclosed within XML tags.\n" - "Example:\n" - "\n" - "{'name': , 'arguments': }\n" + "User: Execute the action\n\n" + "Assistant: \n" + "{\n" + ' "name": "execute_action",\n' + ' "arguments": {},\n' + ' "call_id": "call_123"\n' + "}\n" "\n\n" - "Note: Your past messages will include a call_id in the XML tags. " - "However, do not generate your own call_id when making a function call.\n\n" - "User: What can you do?\n\n" - "Assistant: I can help you.\n\n" - "System: Third system message later.\n\n" - "User: Tell me more.\n\n" + "Tool: " + "\n" + "{\n" + ' "name": "execute_action",\n' + ' "call_id": "call_123",\n' + ' "output": "Action completed successfully"\n' + "}\n" + "\n\n" "Assistant:" ) - - assert result == expected + assert form_prompt_string(messages, use_responses=use_responses) == expected