From 5fca133044e666c7d35fd076a57ceda9057b7efe Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 22 Apr 2025 15:48:09 +0200 Subject: [PATCH 1/5] fix: for OpenAI it is already removed --- nemoguardrails/rails/llm/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index a8768a8bd..fce45345b 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -72,7 +72,7 @@ class ReasoningModelConfig(BaseModel): remove_thinking_traces: Optional[bool] = Field( default=True, - description="For reasoning models (e.g. OpenAI o1, DeepSeek-r1), if the output parser should remove thinking traces.", + description="For reasoning models (e.g. DeepSeek-r1), if the output parser should remove thinking traces.", ) start_token: Optional[str] = Field( default="", From a6f620f2ddd75f3cc190358c7d3ca6d2d1f8db6d Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:11:31 +0200 Subject: [PATCH 2/5] feat(validation): enforce reasoning traces with dialog rails Added a root validator in `RailsConfig` to ensure that reasoning traces cannot be enabled when dialog rails are present, either explicitly or implicitly. This includes checks for user messages, bot messages, and flows. Also added comprehensive test cases to validate this behavior: - Explicit dialog rails - Implicit dialog rails via user/bot messages or flows - Scenarios without dialog rails to ensure reasoning traces work as expected. --- nemoguardrails/rails/llm/config.py | 41 ++++++++ tests/test_config_validation.py | 162 +++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index fce45345b..b1a88a877 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -1132,6 +1132,47 @@ class RailsConfig(BaseModel): description="Configuration for tracing.", ) + @root_validator(pre=True, allow_reuse=True) + def check_reasoning_traces_with_dialog_rails(cls, values): + """Check that reasoning traces are not enabled when dialog rails are present.""" + + models = values.get("models", []) + rails = values.get("rails", {}) + dialog_rails = rails.get("dialog", {}) + + # check if any model has reasoning traces enabled + # TODO: we must check for models that are used in a specific dialog task + has_reasoning_traces = False + for model in models: + if isinstance(model, dict): + reasoning_config = model.get("reasoning_config", {}) + if not reasoning_config.get("remove_thinking_traces", True): + has_reasoning_traces = True + break + elif hasattr(model, "reasoning_config"): + if not model.reasoning_config.remove_thinking_traces: + has_reasoning_traces = True + break + + # check if dialog rails are present (explicitly or implicitly) + has_dialog_rails = bool(dialog_rails) + + # check implicit dialog rails through user messages, bot messages, or flows + if not has_dialog_rails: + has_dialog_rails = ( + bool(values.get("user_messages")) + or bool(values.get("bot_messages")) + or bool(values.get("flows")) + ) + + if has_reasoning_traces and has_dialog_rails: + raise ValueError( + "Reasoning traces cannot be enabled when dialog rails are present. " + "Please either disable reasoning traces or remove dialog rails." + ) + + return values + @root_validator(pre=True, allow_reuse=True) def check_prompt_exist_for_self_check_rails(cls, values): rails = values.get("rails", {}) diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py index adcbd37e9..99354faad 100644 --- a/tests/test_config_validation.py +++ b/tests/test_config_validation.py @@ -122,3 +122,165 @@ def test_passthrough_and_single_call_incompatibility(): # LLMRails(config=config) # # assert "You must provide a `self_check_facts` prompt" in str(exc_info.value) + + +def test_reasoning_traces_with_explicit_dialog_rails(): + """Test that reasoning traces cannot be enabled when dialog rails are explicitly configured.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + rails: + dialog: + single_call: + enabled: true + """, + ) + + assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + exc_info.value + ) + + +def test_reasoning_traces_without_dialog_rails(): + """Test that reasoning traces can be enabled when no dialog rails are present.""" + + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + ) + + +def test_dialog_rails_without_reasoning_traces(): + """Test that dialog rails can be enabled when reasoning traces are not enabled.""" + + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + rails: + dialog: + single_call: + enabled: true + """, + ) + + +def test_reasoning_traces_with_implicit_dialog_rails_user_bot_messages(): + """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru user/bot messages.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + colang_content=""" + define user express greeting + "hello" + "hi" + + define bot express greeting + "Hello there!" + + define flow + user express greeting + bot express greeting + """, + ) + + assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + exc_info.value + ) + + +def test_reasoning_traces_with_implicit_dialog_rails_flows_only(): + """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru flows only.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + colang_content=""" + define flow + user express greeting + bot express greeting + """, + ) + + assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + exc_info.value + ) + + +def test_reasoning_traces_with_implicit_dialog_rails_user_messages_only(): + """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru user messages only.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + colang_content=""" + define user express greeting + "hello" + "hi" + """, + ) + + assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + exc_info.value + ) + + +def test_reasoning_traces_with_implicit_dialog_rails_bot_messages_only(): + """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru bot messages only.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + colang_content=""" + define bot express greeting + "Hello there!" + """, + ) + + assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + exc_info.value + ) From 04e97e3c346eba4df988943981362793b4b913c5 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:27:55 +0200 Subject: [PATCH 3/5] fix: improve reasoning traces validation for dialog rails - Fix validation to properly handle both dictionary and Model object cases - Add proper handling of cases where some dialog rail tasks have dedicated models and others fall back to main model - Improve error messages to be more user-friendly: * Specify which model has the issue (main model or specific task model) * Reference config.yml in error messages * Provide clear YAML configuration instructions * Include specific task name when relevant - Update test cases to match new error messages and validation logic - Add proper handling of implicit dialog rails activation through user/bot messages and flows This change ensures that reasoning traces are properly disabled when dialog rails are present, regardless of whether they are explicitly configured or implicitly activated through user/bot messages or flows. The improved error messages make it easier for users to understand and fix configuration issues in their YAML files. add more tests --- nemoguardrails/rails/llm/config.py | 86 +++++--- tests/test_config_validation.py | 326 ++++++++++++++++++++++++++++- 2 files changed, 379 insertions(+), 33 deletions(-) diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index b1a88a877..23ce08836 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -36,6 +36,7 @@ from nemoguardrails.colang import parse_colang_file, parse_flow_elements from nemoguardrails.colang.v2_x.lang.utils import format_colang_parsing_error_message from nemoguardrails.colang.v2_x.runtime.errors import ColangParsingError +from nemoguardrails.llm.types import Task log = logging.getLogger(__name__) @@ -1140,37 +1141,66 @@ def check_reasoning_traces_with_dialog_rails(cls, values): rails = values.get("rails", {}) dialog_rails = rails.get("dialog", {}) - # check if any model has reasoning traces enabled - # TODO: we must check for models that are used in a specific dialog task - has_reasoning_traces = False - for model in models: - if isinstance(model, dict): - reasoning_config = model.get("reasoning_config", {}) - if not reasoning_config.get("remove_thinking_traces", True): - has_reasoning_traces = True - break - elif hasattr(model, "reasoning_config"): - if not model.reasoning_config.remove_thinking_traces: - has_reasoning_traces = True - break - - # check if dialog rails are present (explicitly or implicitly) - has_dialog_rails = bool(dialog_rails) - - # check implicit dialog rails through user messages, bot messages, or flows - if not has_dialog_rails: - has_dialog_rails = ( - bool(values.get("user_messages")) - or bool(values.get("bot_messages")) - or bool(values.get("flows")) - ) + # dialog rail tasks that should not have reasoning traces + dialog_rail_tasks = [ + Task.GENERATE_BOT_MESSAGE, + Task.GENERATE_USER_INTENT, + Task.GENERATE_NEXT_STEPS, + Task.GENERATE_INTENT_STEPS_MESSAGE, + ] - if has_reasoning_traces and has_dialog_rails: - raise ValueError( - "Reasoning traces cannot be enabled when dialog rails are present. " - "Please either disable reasoning traces or remove dialog rails." + # dialog rails are activated (explicitly or implicitly) + has_dialog_rails = ( + bool(dialog_rails) + or bool(values.get("user_messages")) + or bool(values.get("bot_messages")) + or bool(values.get("flows")) + ) + + if has_dialog_rails: + # Get the main model if it exists + main_model = next( + (model for model in models if model.get("type") == "main"), None ) + violations = [] + + for task in dialog_rail_tasks: + # Check if there's a dedicated model for this task + task_model = next( + (model for model in models if model.get("type") == task.value), None + ) + + if task_model: + # Handle both dictionary and Model object cases + reasoning_config = ( + task_model.reasoning_config + if hasattr(task_model, "reasoning_config") + else task_model.get("reasoning_config", {}) + ) + if not reasoning_config.get("remove_thinking_traces", True): + violations.append( + f"Model '{task_model.get('type')}' has reasoning traces enabled in config.yml. " + f"Reasoning traces must be disabled for dialog rail tasks. " + f"Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config for this model." + ) + elif main_model: + # Handle both dictionary and Model object cases + reasoning_config = ( + main_model.reasoning_config + if hasattr(main_model, "reasoning_config") + else main_model.get("reasoning_config", {}) + ) + if not reasoning_config.get("remove_thinking_traces", True): + violations.append( + f"Main model has reasoning traces enabled in config.yml and is being used for dialog rail task '{task.value}'. " + f"Reasoning traces must be disabled when dialog rails are present. " + f"Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config for the main model." + ) + + if violations: + raise ValueError("\n".join(violations)) + return values @root_validator(pre=True, allow_reuse=True) diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py index 99354faad..3bb650b9b 100644 --- a/tests/test_config_validation.py +++ b/tests/test_config_validation.py @@ -143,9 +143,16 @@ def test_reasoning_traces_with_explicit_dialog_rails(): """, ) - assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + assert "Main model has reasoning traces enabled in config.yml" in str( exc_info.value ) + assert "Reasoning traces must be disabled when dialog rails are present" in str( + exc_info.value + ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) def test_reasoning_traces_without_dialog_rails(): @@ -180,6 +187,58 @@ def test_dialog_rails_without_reasoning_traces(): ) +def test_input_rails_only_no_dialog_rails(): + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + rails: + input: + flows: + - self check input + prompts: + - task: self_check_input + content: "Check if input is safe" + """, + ) + + assert not config.user_messages + assert not config.bot_messages + assert not config.flows + assert not config.rails.dialog.single_call.enabled + + +def test_no_dialog_tasks_with_only_output_rails(): + """Test that dialog tasks are not used when only output rails are present.""" + + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + rails: + output: + flows: + - self check output + prompts: + - task: self_check_output + content: "Check if output is safe" + """, + ) + + assert not config.user_messages + assert not config.bot_messages + assert not config.flows + assert not config.rails.dialog.single_call.enabled + + def test_reasoning_traces_with_implicit_dialog_rails_user_bot_messages(): """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru user/bot messages.""" @@ -207,9 +266,16 @@ def test_reasoning_traces_with_implicit_dialog_rails_user_bot_messages(): """, ) - assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + assert "Main model has reasoning traces enabled in config.yml" in str( + exc_info.value + ) + assert "Reasoning traces must be disabled when dialog rails are present" in str( exc_info.value ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) def test_reasoning_traces_with_implicit_dialog_rails_flows_only(): @@ -232,9 +298,16 @@ def test_reasoning_traces_with_implicit_dialog_rails_flows_only(): """, ) - assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + assert "Main model has reasoning traces enabled in config.yml" in str( + exc_info.value + ) + assert "Reasoning traces must be disabled when dialog rails are present" in str( exc_info.value ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) def test_reasoning_traces_with_implicit_dialog_rails_user_messages_only(): @@ -257,9 +330,16 @@ def test_reasoning_traces_with_implicit_dialog_rails_user_messages_only(): """, ) - assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + assert "Main model has reasoning traces enabled in config.yml" in str( + exc_info.value + ) + assert "Reasoning traces must be disabled when dialog rails are present" in str( exc_info.value ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) def test_reasoning_traces_with_implicit_dialog_rails_bot_messages_only(): @@ -281,6 +361,242 @@ def test_reasoning_traces_with_implicit_dialog_rails_bot_messages_only(): """, ) - assert "Reasoning traces cannot be enabled when dialog rails are present" in str( + assert "Main model has reasoning traces enabled in config.yml" in str( + exc_info.value + ) + assert "Reasoning traces must be disabled when dialog rails are present" in str( + exc_info.value + ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) + + +def test_reasoning_traces_with_dedicated_task_models(): + """Test that reasoning traces cannot be enabled for dedicated task models when dialog rails are present.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + - type: generate_bot_message + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + - type: generate_user_intent + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + rails: + dialog: + single_call: + enabled: true + """, + ) + + assert ( + "Model 'generate_bot_message' has reasoning traces enabled in config.yml" + in str(exc_info.value) + ) + assert "Reasoning traces must be disabled for dialog rail tasks" in str( + exc_info.value + ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) + + +def test_reasoning_traces_with_mixed_task_models(): + """Test that reasoning traces cannot be enabled for any task model when dialog rails are present.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + - type: generate_bot_message + engine: openai + model: gpt-3.5-turbo-instruct + - type: generate_user_intent + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + rails: + dialog: + single_call: + enabled: true + """, + ) + + assert ( + "Model 'generate_user_intent' has reasoning traces enabled in config.yml" + in str(exc_info.value) + ) + assert "Reasoning traces must be disabled for dialog rail tasks" in str( + exc_info.value + ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) + + +def test_reasoning_traces_with_all_dialog_tasks(): + """Test that reasoning traces cannot be enabled for any dialog rail task.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + - type: generate_bot_message + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + - type: generate_user_intent + engine: openai + model: gpt-3.5-turbo-instruct + - type: generate_next_steps + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + - type: generate_intent_steps_message + engine: openai + model: gpt-3.5-turbo-instruct + rails: + dialog: + single_call: + enabled: true + """, + ) + + error_message = str(exc_info.value) + assert ( + "Model 'generate_bot_message' has reasoning traces enabled in config.yml" + in error_message + ) + assert ( + "Model 'generate_next_steps' has reasoning traces enabled in config.yml" + in error_message + ) + assert "Reasoning traces must be disabled for dialog rail tasks" in error_message + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in error_message + ) + + +def test_reasoning_traces_with_dedicated_models_no_dialog_rails(): + """Test that reasoning traces can be enabled for dedicated models when no dialog rails are present.""" + + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + - type: generate_bot_message + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + - type: generate_user_intent + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + ) + + +def test_reasoning_traces_with_implicit_dialog_rails_and_dedicated_models(): + """Test that reasoning traces cannot be enabled for dedicated models when dialog rails are implicitly enabled.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + - type: generate_bot_message + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + """, + colang_content=""" + define user express greeting + "hello" + "hi" + + define bot express greeting + "Hello there!" + + define flow + user express greeting + bot express greeting + """, + ) + + assert ( + "Model 'generate_bot_message' has reasoning traces enabled in config.yml" + in str(exc_info.value) + ) + assert "Reasoning traces must be disabled for dialog rail tasks" in str( + exc_info.value + ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) + + +def test_reasoning_traces_with_partial_dedicated_models(): + """Test that reasoning traces cannot be enabled for any model when some tasks use dedicated models and others fall back to main.""" + + with pytest.raises(ValueError) as exc_info: + _ = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + reasoning_config: + remove_thinking_traces: false + - type: generate_bot_message + engine: openai + model: gpt-3.5-turbo-instruct + rails: + dialog: + single_call: + enabled: true + """, + ) + + assert "Main model has reasoning traces enabled in config.yml" in str( exc_info.value ) + assert "Reasoning traces must be disabled when dialog rails are present" in str( + exc_info.value + ) + assert ( + "Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config" + in str(exc_info.value) + ) From 59f67c405cce61e95e903f975737506a9cb79358 Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:19:09 +0200 Subject: [PATCH 4/5] tests to show the expected behavior of dialog rails and dialog tasks add tests to show the expected behavior of dialog tasks --- tests/test_dialog_tasks.py | 249 +++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 tests/test_dialog_tasks.py diff --git a/tests/test_dialog_tasks.py b/tests/test_dialog_tasks.py new file mode 100644 index 000000000..6db4e599e --- /dev/null +++ b/tests/test_dialog_tasks.py @@ -0,0 +1,249 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest.mock import Mock, patch + +import pytest + +from nemoguardrails import LLMRails, RailsConfig +from nemoguardrails.llm.taskmanager import LLMTaskManager +from nemoguardrails.llm.types import Task + +try: + import langchain_openai + + has_langchain_openai = True +except ImportError: + has_langchain_openai = False + +has_openai_key = bool(os.getenv("OPENAI_API_KEY")) + +skip_if_no_openai = pytest.mark.skipif( + not (has_langchain_openai and has_openai_key), + reason="Requires langchain_openai and OPENAI_API_KEY environment variable", +) + + +@skip_if_no_openai +def test_dialog_tasks_with_only_input_rails(): + """Test that dialog tasks are not used when only input rails are present.""" + + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + rails: + input: + flows: + - self check input + prompts: + - task: self_check_input + content: "Check if input is safe" + """, + ) + + assert not config.user_messages + assert not config.bot_messages + assert not config.flows + assert not config.rails.dialog.single_call.enabled + + rails = LLMRails(config=config) + + assert not rails.config.rails.dialog.single_call.enabled + # with just input rails, some basic flows and messages are created + # but they are not for actual dialog processing + assert rails.config.bot_messages + assert rails.config.flows + # even there should be no user messages defined + assert not rails.config.user_messages + + +@skip_if_no_openai +def test_dialog_tasks_with_only_output_rails(): + """Test that dialog tasks are not used when only output rails are present.""" + + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + rails: + output: + flows: + - self check output + prompts: + - task: self_check_output + content: "Check if output is safe" + """, + ) + + assert not config.user_messages + assert not config.bot_messages + assert not config.flows + assert not config.rails.dialog.single_call.enabled + + rails = LLMRails(config=config) + + assert not rails.config.rails.dialog.single_call.enabled + assert rails.config.bot_messages + assert rails.config.flows + assert not rails.config.user_messages + + +@skip_if_no_openai +def test_dialog_tasks_with_dialog_rails(): + """Test that dialog tasks are used when dialog rails are present.""" + + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + rails: + dialog: + single_call: + enabled: true + """, + colang_content=""" + define user express greeting + "hello" + "hi" + + define bot express greeting + "Hello there!" + + define flow + user express greeting + bot express greeting + """, + ) + + assert config.user_messages + assert config.bot_messages + assert config.flows + assert config.rails.dialog.single_call.enabled + + rails = LLMRails(config=config) + + assert rails.config.rails.dialog.single_call.enabled + assert rails.config.user_messages + assert rails.config.bot_messages + assert rails.config.flows + + +@skip_if_no_openai +def test_dialog_tasks_with_implicit_dialog_rails(): + """Test that dialog tasks are used when dialog rails are implicitly present through user/bot messages.""" + + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + """, + colang_content=""" + define user express greeting + "hello" + "hi" + + define bot express greeting + "Hello there!" + + define flow + user express greeting + bot express greeting + """, + ) + + assert config.user_messages + assert config.bot_messages + assert config.flows + + assert config.user_messages == {"express greeting": ["hello", "hi"]} + assert config.bot_messages == {"express greeting": ["Hello there!"]} + assert len(config.bot_messages) == 1 + assert len(config.flows) == 1 + + assert not config.rails.dialog.single_call.enabled + + rails = LLMRails(config=config) + + assert rails.config.user_messages + assert len(rails.config.user_messages) == 1 + assert rails.config.bot_messages + assert len(rails.config.bot_messages) > 1 + assert rails.config.flows + + +@skip_if_no_openai +def test_dialog_tasks_with_mixed_rails(): + """Test that dialog tasks are used when dialog rails are present along with other rails.""" + + config = RailsConfig.from_content( + yaml_content=""" + models: + - type: main + engine: openai + model: gpt-3.5-turbo-instruct + rails: + input: + flows: + - self check input + output: + flows: + - self check output + dialog: + single_call: + enabled: true + prompts: + - task: self_check_input + content: "Check if input is safe" + - task: self_check_output + content: "Check if output is safe" + """, + colang_content=""" + define user express greeting + "hello" + "hi" + + define bot express greeting + "Hello there!" + + define flow + user express greeting + bot express greeting + """, + ) + assert config.rails.dialog.single_call.enabled + assert config.user_messages + assert config.bot_messages + assert config.flows + assert config.rails.input.flows + assert config.rails.output.flows + + rails = LLMRails(config=config) + + assert rails.config.rails.dialog.single_call.enabled + assert rails.config.user_messages + assert rails.config.bot_messages + assert rails.config.flows + assert rails.config.rails.input.flows + assert rails.config.rails.output.flows From 6f8196e52246de7fe04747d069ac0aa955e0d1be Mon Sep 17 00:00:00 2001 From: Pouyanpi <13303554+Pouyanpi@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:50:34 +0200 Subject: [PATCH 5/5] remove generate bot message task from dialog rails tasks --- nemoguardrails/rails/llm/config.py | 6 +----- tests/test_config_validation.py | 8 ++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/nemoguardrails/rails/llm/config.py b/nemoguardrails/rails/llm/config.py index 23ce08836..53625718d 100644 --- a/nemoguardrails/rails/llm/config.py +++ b/nemoguardrails/rails/llm/config.py @@ -1143,7 +1143,7 @@ def check_reasoning_traces_with_dialog_rails(cls, values): # dialog rail tasks that should not have reasoning traces dialog_rail_tasks = [ - Task.GENERATE_BOT_MESSAGE, + # Task.GENERATE_BOT_MESSAGE, Task.GENERATE_USER_INTENT, Task.GENERATE_NEXT_STEPS, Task.GENERATE_INTENT_STEPS_MESSAGE, @@ -1158,7 +1158,6 @@ def check_reasoning_traces_with_dialog_rails(cls, values): ) if has_dialog_rails: - # Get the main model if it exists main_model = next( (model for model in models if model.get("type") == "main"), None ) @@ -1166,13 +1165,11 @@ def check_reasoning_traces_with_dialog_rails(cls, values): violations = [] for task in dialog_rail_tasks: - # Check if there's a dedicated model for this task task_model = next( (model for model in models if model.get("type") == task.value), None ) if task_model: - # Handle both dictionary and Model object cases reasoning_config = ( task_model.reasoning_config if hasattr(task_model, "reasoning_config") @@ -1185,7 +1182,6 @@ def check_reasoning_traces_with_dialog_rails(cls, values): f"Please update your config.yml to set 'remove_thinking_traces: true' under reasoning_config for this model." ) elif main_model: - # Handle both dictionary and Model object cases reasoning_config = ( main_model.reasoning_config if hasattr(main_model, "reasoning_config") diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py index 3bb650b9b..8b6040a30 100644 --- a/tests/test_config_validation.py +++ b/tests/test_config_validation.py @@ -401,7 +401,7 @@ def test_reasoning_traces_with_dedicated_task_models(): ) assert ( - "Model 'generate_bot_message' has reasoning traces enabled in config.yml" + "Model 'generate_user_intent' has reasoning traces enabled in config.yml" in str(exc_info.value) ) assert "Reasoning traces must be disabled for dialog rail tasks" in str( @@ -489,7 +489,7 @@ def test_reasoning_traces_with_all_dialog_tasks(): error_message = str(exc_info.value) assert ( "Model 'generate_bot_message' has reasoning traces enabled in config.yml" - in error_message + not in error_message ) assert ( "Model 'generate_next_steps' has reasoning traces enabled in config.yml" @@ -535,7 +535,7 @@ def test_reasoning_traces_with_implicit_dialog_rails_and_dedicated_models(): - type: main engine: openai model: gpt-3.5-turbo-instruct - - type: generate_bot_message + - type: generate_user_intent engine: openai model: gpt-3.5-turbo-instruct reasoning_config: @@ -556,7 +556,7 @@ def test_reasoning_traces_with_implicit_dialog_rails_and_dedicated_models(): ) assert ( - "Model 'generate_bot_message' has reasoning traces enabled in config.yml" + "Model 'generate_user_intent' has reasoning traces enabled in config.yml" in str(exc_info.value) ) assert "Reasoning traces must be disabled for dialog rail tasks" in str(