feat(validation): enforce reasoning traces with dialog rails

Added a root validator in `RailsConfig` to ensure that reasoning traces
cannot be enabled when dialog rails are present, either explicitly or
implicitly. This includes checks for user messages, bot messages, and
flows.

Also added comprehensive test cases to validate this behavior:
- Explicit dialog rails
- Implicit dialog rails via user/bot messages or flows
- Scenarios without dialog rails to ensure reasoning traces work as
  expected.

Author: Pouyanpi

nemoguardrails/rails/llm/config.py

@@ -1133,6 +1133,47 @@ class RailsConfig(BaseModel):
         description="Configuration for tracing.",
     )
 
+    @root_validator(pre=True, allow_reuse=True)
+    def check_reasoning_traces_with_dialog_rails(cls, values):
+        """Check that reasoning traces are not enabled when dialog rails are present."""
+
+        models = values.get("models", [])
+        rails = values.get("rails", {})
+        dialog_rails = rails.get("dialog", {})
+
+        # check if any model has reasoning traces enabled
+        # TODO: we must check for models that are used in a specific dialog task
+        has_reasoning_traces = False
+        for model in models:
+            if isinstance(model, dict):
+                reasoning_config = model.get("reasoning_config", {})
+                if not reasoning_config.get("remove_thinking_traces", True):
+                    has_reasoning_traces = True
+                    break
+            elif hasattr(model, "reasoning_config"):
+                if not model.reasoning_config.remove_thinking_traces:
+                    has_reasoning_traces = True
+                    break
+
+        # check if dialog rails are present (explicitly or implicitly)
+        has_dialog_rails = bool(dialog_rails)
+
+        # check implicit dialog rails through user messages, bot messages, or flows
+        if not has_dialog_rails:
+            has_dialog_rails = (
+                bool(values.get("user_messages"))
+                or bool(values.get("bot_messages"))
+                or bool(values.get("flows"))
+            )
+
+        if has_reasoning_traces and has_dialog_rails:
+            raise ValueError(
+                "Reasoning traces cannot be enabled when dialog rails are present. "
+                "Please either disable reasoning traces or remove dialog rails."
+            )
+
+        return values
+
     @root_validator(pre=True, allow_reuse=True)
     def check_prompt_exist_for_self_check_rails(cls, values):
         rails = values.get("rails", {})

tests/test_config_validation.py

@@ -122,3 +122,165 @@ def test_passthrough_and_single_call_incompatibility():
 #         LLMRails(config=config)
 #
 #     assert "You must provide a `self_check_facts` prompt" in str(exc_info.value)
+
+
+def test_reasoning_traces_with_explicit_dialog_rails():
+    """Test that reasoning traces cannot be enabled when dialog rails are explicitly configured."""
+
+    with pytest.raises(ValueError) as exc_info:
+        _ = RailsConfig.from_content(
+            yaml_content="""
+            models:
+              - type: main
+                engine: openai
+                model: gpt-3.5-turbo-instruct
+                reasoning_config:
+                  remove_thinking_traces: false
+            rails:
+              dialog:
+                single_call:
+                  enabled: true
+            """,
+        )
+
+    assert "Reasoning traces cannot be enabled when dialog rails are present" in str(
+        exc_info.value
+    )
+
+
+def test_reasoning_traces_without_dialog_rails():
+    """Test that reasoning traces can be enabled when no dialog rails are present."""
+
+    _ = RailsConfig.from_content(
+        yaml_content="""
+        models:
+          - type: main
+            engine: openai
+            model: gpt-3.5-turbo-instruct
+            reasoning_config:
+              remove_thinking_traces: false
+        """,
+    )
+
+
+def test_dialog_rails_without_reasoning_traces():
+    """Test that dialog rails can be enabled when reasoning traces are not enabled."""
+
+    _ = RailsConfig.from_content(
+        yaml_content="""
+        models:
+          - type: main
+            engine: openai
+            model: gpt-3.5-turbo-instruct
+        rails:
+          dialog:
+            single_call:
+              enabled: true
+        """,
+    )
+
+
+def test_reasoning_traces_with_implicit_dialog_rails_user_bot_messages():
+    """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru user/bot messages."""
+
+    with pytest.raises(ValueError) as exc_info:
+        _ = RailsConfig.from_content(
+            yaml_content="""
+            models:
+              - type: main
+                engine: openai
+                model: gpt-3.5-turbo-instruct
+                reasoning_config:
+                  remove_thinking_traces: false
+            """,
+            colang_content="""
+            define user express greeting
+              "hello"
+              "hi"
+
+            define bot express greeting
+              "Hello there!"
+
+            define flow
+              user express greeting
+              bot express greeting
+            """,
+        )
+
+    assert "Reasoning traces cannot be enabled when dialog rails are present" in str(
+        exc_info.value
+    )
+
+
+def test_reasoning_traces_with_implicit_dialog_rails_flows_only():
+    """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru flows only."""
+
+    with pytest.raises(ValueError) as exc_info:
+        _ = RailsConfig.from_content(
+            yaml_content="""
+            models:
+              - type: main
+                engine: openai
+                model: gpt-3.5-turbo-instruct
+                reasoning_config:
+                  remove_thinking_traces: false
+            """,
+            colang_content="""
+            define flow
+              user express greeting
+              bot express greeting
+            """,
+        )
+
+    assert "Reasoning traces cannot be enabled when dialog rails are present" in str(
+        exc_info.value
+    )
+
+
+def test_reasoning_traces_with_implicit_dialog_rails_user_messages_only():
+    """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru user messages only."""
+
+    with pytest.raises(ValueError) as exc_info:
+        _ = RailsConfig.from_content(
+            yaml_content="""
+            models:
+              - type: main
+                engine: openai
+                model: gpt-3.5-turbo-instruct
+                reasoning_config:
+                  remove_thinking_traces: false
+            """,
+            colang_content="""
+            define user express greeting
+              "hello"
+              "hi"
+            """,
+        )
+
+    assert "Reasoning traces cannot be enabled when dialog rails are present" in str(
+        exc_info.value
+    )
+
+
+def test_reasoning_traces_with_implicit_dialog_rails_bot_messages_only():
+    """Test that reasoning traces cannot be enabled when dialog rails are implicitly enabled thru bot messages only."""
+
+    with pytest.raises(ValueError) as exc_info:
+        _ = RailsConfig.from_content(
+            yaml_content="""
+            models:
+              - type: main
+                engine: openai
+                model: gpt-3.5-turbo-instruct
+                reasoning_config:
+                  remove_thinking_traces: false
+            """,
+            colang_content="""
+            define bot express greeting
+              "Hello there!"
+            """,
+        )
+
+    assert "Reasoning traces cannot be enabled when dialog rails are present" in str(
+        exc_info.value
+    )