allow for missing kafka/websocket C++ adapters, enable future mmissing parquet adapter after cache cleanup

Signed-off-by: Tim Paine <[email protected]>

Raise hard import errors if trying to use adapters with missing dependencies, with helpful error messages. Remove runtime requirement on pandas

Signed-off-by: Tim Paine <[email protected]>

Remove superfluous checks

Signed-off-by: Tim Paine <[email protected]>

Add oldest supported numpy as runtime dep as well

Signed-off-by: Tim Paine <[email protected]>

fix spacing in action, dont depend on oldest-supported-numpy

Signed-off-by: Tim Paine <[email protected]>

Isolate sdist build environment

Signed-off-by: Tim Paine <[email protected]>

Author: timkpaine

.github/actions/setup-caches/action.yml

@@ -61,7 +61,7 @@ runs:
           /Users/runner/vcpkg_download_cache
         key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }}
         restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}-
-      if: ${{ runner.os == 'macOS' && inputs.vcpkg == 'true'}}
+      if: ${{ runner.os == 'macOS' && inputs.vcpkg == 'true' }}
 
     - name: Setup vcpkg cache in shell (Windows)
       shell: bash
@@ -70,7 +70,7 @@ runs:
         mkdir C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache
         echo "VCPKG_DEFAULT_BINARY_CACHE=C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache" >> $GITHUB_ENV
         echo "VCPKG_DOWNLOADS=C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache" >> $GITHUB_ENV
-      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true'}}
+      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true' }}
 
     - name: Setup vcpkg cache (Windows)
       uses: actions/cache@v4
@@ -80,4 +80,4 @@ runs:
           C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache
         key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }}
         restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}-
-      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true'}}
+      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true' }}

Makefile

@@ -142,7 +142,7 @@ major:
 dist-py: dist-py-sdist  # Build python dist
 dist-py-sdist:
 	rm -rf csp/lib/*
-	python -m build --sdist -n
+	python -m build --sdist
 
 dist-py-wheel:
 	python setup.py bdist_wheel $(EXTRA_ARGS)

csp/adapters/db.py

@@ -26,11 +26,8 @@
         _SQLALCHEMY_2 = False
 
     import sqlalchemy as db
-
-    _HAS_SQLALCHEMY = True
 except (PackageNotFoundError, ValueError, TypeError, ImportError):
-    _HAS_SQLALCHEMY = False
-    db = None
+    raise ModuleNotFoundError("csp's db adapter requires `sqlalchemy`")
 
 
 class TimeAccessor(ABC):
@@ -200,8 +197,6 @@ def __init__(
         :param log_query: set to True to see what query was generated to access the data
         :param use_raw_user_query: Don't do any alteration to user query, assume it contains all the needed columns and sorting
         """
-        if not _HAS_SQLALCHEMY:
-            raise RuntimeError("Could not find SQLAlchemy installation")
         self._connection = connection
         self._table_name = table_name
         self._schema_name = schema_name

csp/adapters/kafka.py

@@ -18,7 +18,11 @@
     hash_mutable,
 )
 from csp.impl.wiring import ReplayMode, input_adapter_def, output_adapter_def, status_adapter_def
-from csp.lib import _kafkaadapterimpl
+
+try:
+    from csp.lib import _kafkaadapterimpl
+except ImportError:
+    raise ImportError("csp's kafka adapter requires the C++ csp extension to be built, but it could not be imported")
 
 _ = BytesMessageProtoMapper, DateTimeType, JSONTextMessageMapper, RawBytesMessageMapper, RawTextMessageMapper
 T = TypeVar("T")

csp/adapters/parquet.py

@@ -16,7 +16,12 @@
 from csp.impl.types.typing_utils import CspTypingUtils
 from csp.impl.wiring import input_adapter_def, status_adapter_def
 from csp.impl.wiring.node import node
-from csp.lib import _parquetadapterimpl
+
+try:
+    from csp.lib import _parquetadapterimpl
+except ImportError:
+    raise ImportError("csp's parquet adapter requires the C++ csp extension to be built, but it could not be imported")
+
 
 __all__ = [
     "ParquetOutputConfig",

csp/adapters/perspective.py

@@ -19,7 +19,7 @@
     import tornado.web
     import tornado.websocket
 except ImportError:
-    raise ImportError("perspective adapter requires tornado package")
+    raise ModuleNotFoundError("csp's perspective adapter requires `tornado`")
 
 
 _PERSPECTIVE_3 = is_perspective3()

csp/adapters/websocket.py

@@ -19,7 +19,13 @@
 )
 from csp.impl.wiring import input_adapter_def, output_adapter_def, status_adapter_def
 from csp.impl.wiring.delayed_node import DelayedNodeWrapperDef
-from csp.lib import _websocketadapterimpl
+
+try:
+    from csp.lib import _websocketadapterimpl
+except ImportError:
+    raise ImportError(
+        "csp's websocket adapter requires the C++ csp extension to be built, but it could not be imported"
+    )
 
 from .websocket_types import WebsocketHeaderUpdate
 
@@ -38,14 +44,14 @@
     import tornado.web
     import tornado.websocket
 except ImportError:
-    raise ImportError("websocket adapter requires tornado package")
+    raise ModuleNotFoundError("csp's websocket adapter requires `tornado`")
 
 try:
     import rapidjson
 
     datetime_mode = rapidjson.DM_UNIX_TIME | rapidjson.DM_NAIVE_IS_UTC
 except ImportError:
-    raise ImportError("websocket adapter requires rapidjson package")
+    raise ModuleNotFoundError("csp's websocket adapter requires `rapidjson`")
 
 
 def diff_dict(old, new):

csp/impl/perspective_common.py

@@ -10,12 +10,12 @@
         _PERSPECTIVE_3 = True
         from perspective.widget import PerspectiveWidget
 
-    elif version.parse(perspective.__version__) >= version.parse("0.6.2"):
+    elif version.parse(perspective.__version__) >= version.parse("2.0.0"):
         from perspective import PerspectiveManager, PerspectiveWidget  # noqa F401
 
         _PERSPECTIVE_3 = False
     else:
-        raise ImportError("perspective adapter requires 0.6.2 or greater of the perspective-python package")
+        raise ImportError("perspective adapter requires 2.0 or greater of the perspective-python package")
 
 except ImportError:
     raise ImportError(

pyproject.toml

@@ -23,14 +23,11 @@ dependencies = [
     "deprecated",
     "numpy<2",
     "packaging",
-    "pandas",
     "psutil",
     "pyarrow>=15,<19",
     "pydantic>=2",
     "pytz",
     "ruamel.yaml",
-    "sqlalchemy",
-    "typing-extensions",
 ]
 
 classifiers = [
@@ -80,6 +77,7 @@ develop = [
     "httpx>=0.20,<1",  # kafka
     "perspective-python>=2",  # perspective
     "ipywidgets",  # perspective
+    "pandas",  # pandas extension
     "polars",  # parquet
     "psutil",  # test_engine/test_history
     "sqlalchemy",  # db
@@ -91,23 +89,6 @@ showgraph = [
     "graphviz",
     "pillow",
 ]
-test = [
-    "graphviz",
-    "pillow",
-    "pytest",
-    "pytest-asyncio",
-    "pytest-cov",
-    "pytest-sugar",
-    "httpx>=0.20,<1",
-    "perspective-python",
-    "polars",
-    "psutil",
-    "requests",
-    "slack-sdk>=3",
-    "sqlalchemy",
-    "threadpoolctl",
-    "tornado",
-]
 symphony = [
     "csp-adapter-symphony",
 ]