chore: fix and revert poetry lock to its stable state (#1133)

Author: Pouyanpi

.github/workflows/latest-deps-tests.yml

@@ -3,6 +3,8 @@ name: Test with Latest Dependencies
 on:
   schedule:
     - cron: "0 22 * * *" # 10:00 PM UTC, daily
+
+  workflow_dispatch:
 jobs:
   latest-deps-tests-matrix:
     strategy:

.github/workflows/test-and-build-wheel.yml

@@ -85,7 +85,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
       - name: Set up Python
         uses: actions/setup-python@v5

poetry.lock

@@ -50,20 +50,8 @@ python = ">=3.9,!=3.9.7,<3.14"
 aiohttp = ">=3.10.11"
 annoy = ">=1.17.3"
 fastapi = ">=0.103.0,"
-fastembed = [
-  { version = ">=0.2.2, <0.5", python = ">=3.9,<3.10" },
-  { version = ">=0.5.0, <=0.6.0", python = ">=3.10, <3.14" },
-]
-onnxruntime = [
-  { version = ">=1.17.0, <=1.19.2", python = ">=3.9, <3.10" },
-  { version = ">=1.20.0", python = ">=3.10, <3.14" },
-]
-
+fastembed = [{ version = ">=0.2.2, <=0.6.0", python = ">=3.9,<3.14" }]
 httpx = ">=0.24.1"
-# onnxruntime in 1.20 has dropped python 3.9 support
-# fastembed has also dropped python 3.9 support entirely as of 0.5.0
-# fastembed depends on onnxruntime we pindown the working version for 3.9 here
-# as it is a transient dependency we must remove it later
 jinja2 = ">=3.1.6"
 langchain = ">=0.2.14,<0.4.0"
 langchain-core = ">=0.2.14,<0.4.0"
@@ -90,20 +78,19 @@ langchain-openai = { version = ">=0.0.5", optional = true }
 
 # eval
 tqdm = { version = ">=4.65,<5.0", optional = true }
-numpy = [
-  { version = ">=1.21,<1.24", python = ">=3.9,<3.10" },
-  { version = ">=1.24,<2.2.0", python = ">=3.10,<3.14" },
-]
 streamlit = { version = "^1.37.0", optional = true, python = ">=3.9,!=3.9.7,<3.12" }
 pandas = { version = ">=1.4.0,<3", optional = true }
-
-spacy = [
-  { version = ">=3.7.2,<3.8.0", optional = true, python = ">=3.9,<3.13" },
-  # { version = "==3.7.2", optional = true, python = ">=3.13,<3.14" },
-] # Capping spacy to spacy<3.8.0  see https://github.com/explosion/cython-blis/issues/117# poetry 3.8.4 does not exist for python 3.9
-# No spacy support for python 3.13 (https://github.com/explosion/spaCy/issues/13658)
-presidio-analyzer = { version = ">=2.2", optional = true }   # sdd
-presidio-anonymizer = { version = ">=2.2", optional = true }
+numpy = [
+  { version = ">=1.21", python = ">=3.10,<3.12" },
+  { version = ">=1.26", python = ">=3.12,<3.13" },
+  { version = ">=2.1.0", python = ">=3.13" },
+  { version = ">=1.21,<2.1.0", python = "<3.10" },
+]
+# sdd
+# we don't need it and we comment it for sake of documentation
+# spacy = { version = ">=3.4.4,<4.0.0,!=3.7.0", python = "<3.13", optional = true }
+presidio-analyzer = { version = ">=2.2", optional = true, python = "<3.13" }
+presidio-anonymizer = { version = ">=2.2", optional = true, python = "<3.13" }
 
 # nim
 langchain-nvidia-ai-endpoints = { version = ">= 0.2.0", optional = true }
@@ -115,7 +102,7 @@ google-cloud-language = { version = ">=2.14.0", optional = true }
 yara-python = { version = "^4.5.1", optional = true }
 
 [tool.poetry.extras]
-sdd = ["presidio-analyzer", "presidio-anonymizer", "spacy"]
+sdd = ["presidio-analyzer", "presidio-anonymizer"]
 eval = ["tqdm", "numpy", "streamlit"]
 openai = ["langchain-openai"]
 gcp = ["google-cloud-language"]
@@ -128,7 +115,6 @@ jailbreak = ["yara-python"]
 all = [
   "presidio-analyzer",
   "presidio-anonymizer",
-  "spacy",
   "tqdm",
   "numpy",
   "streamlit",

pyproject.toml

@@ -42,7 +42,7 @@ def colang_1_config():
             dialog:
                 user_messages:
                     embeddings_only: True
-                    embeddings_only_similarity_threshold: 0.99
+                    embeddings_only_similarity_threshold: 0.8
                     embeddings_only_fallback_intent: "express greeting"
         """,
     )
@@ -76,7 +76,7 @@ def colang_2_config():
         dialog:
             user_messages:
                 embeddings_only: True
-                embeddings_only_similarity_threshold: 0.99
+                embeddings_only_similarity_threshold: 0.8
                 embeddings_only_fallback_intent: "user expressed greeting"
     """,
     )

tests/test_embeddings_only_user_messages.py

@@ -13,28 +13,60 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# We detect if the environment is set up correct for SDD (presidio + downloaded spacy model)
+# poetry install -e "sdd" --with dev
+# python -m spacy download en_core_web_lg
+
+import subprocess
+
 import pytest
 
 from nemoguardrails import RailsConfig
 from nemoguardrails.actions import action
 from nemoguardrails.actions.actions import ActionResult
 from tests.utils import TestChat
 
-# We detect if the environment is set up correct for SDD (presidio + downloaded spacy model)
-# poetry install -e "sdd" --with dev
-# python -m spacy download en_core_web_lg
 try:
     import presidio_analyzer
     import presidio_anonymizer
     import spacy
 
-    assert spacy.util.is_package("en_core_web_lg")
-
     SDD_SETUP_PRESENT = True
-except (ImportError, AssertionError):
+except ImportError:
     SDD_SETUP_PRESENT = False
 
 
+def setup_module(module):
+    if not SDD_SETUP_PRESENT:
+        pytest.skip("Required dependencies not found")
+
+    try:
+        # check if the model is already downloaded
+        if not spacy.util.is_package("en_core_web_lg"):
+            subprocess.run(
+                ["python", "-m", "spacy", "download", "en_core_web_lg"],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+        # ensure the model is now available
+
+        if not spacy.util.is_package("en_core_web_lg"):
+            pytest.skip("Failed to download or verify spaCy model 'en_core_web_lg'")
+
+    except subprocess.CalledProcessError as e:
+        pytest.skip(f"Error downloading spaCy model: {e.stderr}")
+    except Exception as e:
+        pytest.skip(f"Unexpected error during setup: {str(e)}")
+
+
+def teardown_module(module):
+    """No cleanup needed as the spaCy model is a persistent dependency
+    that should remain available for future test runs."""
+    pass
+
+
 @pytest.mark.skipif(
     not SDD_SETUP_PRESENT, reason="Sensitive Data Detection setup is not present."
 )