allow for missing kafka/websocket C++ adapters, enable future mmissing parquet adapter after cache cleanup

Signed-off-by: Tim Paine <[email protected]>

Raise hard import errors if trying to use adapters with missing dependencies, with helpful error messages. Remove runtime requirement on pandas

Signed-off-by: Tim Paine <[email protected]>

Remove superfluous checks

Signed-off-by: Tim Paine <[email protected]>

Add oldest supported numpy as runtime dep as well

Signed-off-by: Tim Paine <[email protected]>

fix spacing in action, dont depend on oldest-supported-numpy

Signed-off-by: Tim Paine <[email protected]>

Isolate sdist build environment

Signed-off-by: Tim Paine <[email protected]>

Author: timkpaine

.github/actions/setup-caches/action.yml

@@ -61,7 +61,7 @@ runs:
           /Users/runner/vcpkg_download_cache
         key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }}
         restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}-
-      if: ${{ runner.os == 'macOS' && inputs.vcpkg == 'true'}}
+      if: ${{ runner.os == 'macOS' && inputs.vcpkg == 'true' }}
 
     - name: Setup vcpkg cache in shell (Windows)
       shell: bash
@@ -70,7 +70,7 @@ runs:
         mkdir C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache
         echo "VCPKG_DEFAULT_BINARY_CACHE=C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_cache" >> $GITHUB_ENV
         echo "VCPKG_DOWNLOADS=C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache" >> $GITHUB_ENV
-      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true'}}
+      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true' }}
 
     - name: Setup vcpkg cache (Windows)
       uses: actions/cache@v4
@@ -80,4 +80,4 @@ runs:
           C:\\Users\\runneradmin\\AppData\\Local\\vcpkg_download_cache
         key: vcpkg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('vcpkg.json') }}
         restore-keys: vcpkg-${{ runner.os }}-${{ runner.arch }}-
-      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true'}}
+      if: ${{ runner.os == 'Windows' && inputs.vcpkg == 'true' }}

Makefile

@@ -143,7 +143,7 @@ major:
 dist-py: dist-py-sdist  # Build python dist
 dist-py-sdist:
 	rm -rf csp/lib/*
-	python -m build --sdist -n
+	python -m build --sdist 
 
 dist-py-wheel:
 	python setup.py bdist_wheel $(EXTRA_ARGS)

csp/adapters/db.py

@@ -25,11 +25,8 @@
         _SQLALCHEMY_2 = False
 
     import sqlalchemy as db
-
-    _HAS_SQLALCHEMY = True
 except (PackageNotFoundError, ValueError, TypeError, ImportError):
-    _HAS_SQLALCHEMY = False
-    db = None
+    raise ModuleNotFoundError("csp's db adapter requires `sqlalchemy`")
 
 
 class TimeAccessor(ABC):
@@ -199,8 +196,6 @@ def __init__(
         :param log_query: set to True to see what query was generated to access the data
         :param use_raw_user_query: Don't do any alteration to user query, assume it contains all the needed columns and sorting
         """
-        if not _HAS_SQLALCHEMY:
-            raise RuntimeError("Could not find SQLAlchemy installation")
         self._connection = connection
         self._table_name = table_name
         self._schema_name = schema_name

csp/adapters/kafka.py

@@ -15,7 +15,11 @@
     RawTextMessageMapper,
 )
 from csp.impl.wiring import input_adapter_def, output_adapter_def, status_adapter_def
-from csp.lib import _kafkaadapterimpl
+
+try:
+    from csp.lib import _kafkaadapterimpl
+except ImportError:
+    raise ImportError("csp's kafka adapter requires the C++ csp extension to be built, but it could not be imported")
 
 _ = BytesMessageProtoMapper, DateTimeType, JSONTextMessageMapper, RawBytesMessageMapper, RawTextMessageMapper
 T = typing.TypeVar("T")

csp/adapters/parquet.py

@@ -15,7 +15,12 @@
 from csp.impl.types.typing_utils import CspTypingUtils
 from csp.impl.wiring import input_adapter_def, status_adapter_def
 from csp.impl.wiring.node import node
-from csp.lib import _parquetadapterimpl
+
+try:
+    from csp.lib import _parquetadapterimpl
+except ImportError:
+    raise ImportError("csp's parquet adapter requires the C++ csp extension to be built, but it could not be imported")
+
 
 __all__ = [
     "ParquetOutputConfig",

csp/adapters/perspective.py

@@ -11,17 +11,17 @@
     import tornado.web
     import tornado.websocket
 except ImportError:
-    raise ImportError("perspective adapter requires tornado package")
+    raise ModuleNotFoundError("csp's perspective adapter requires `tornado`")
 
 
 try:
     from perspective import PerspectiveManager, Table as Table_, View as View_, __version__, set_threadpool_size
 
     MAJOR, MINOR, PATCH = map(int, __version__.split("."))
     if (MAJOR, MINOR, PATCH) < (0, 6, 2):
-        raise ImportError("perspective adapter requires 0.6.2 or greater of the perspective-python package")
+        raise ModuleNotFoundError("csp's perspective adapter requires `perspective-python>=0.6.2`")
 except ImportError:
-    raise ImportError("perspective adapter requires 0.6.2 or greater of the perspective-python package")
+    raise ModuleNotFoundError("csp's perspective adapter requires `perspective-python>=0.6.2`")
 
 
 # Run perspective update in a separate tornado loop

csp/adapters/slack.py

@@ -20,10 +20,8 @@
     from slack_sdk.socket_mode.request import SocketModeRequest
     from slack_sdk.socket_mode.response import SocketModeResponse
     from slack_sdk.web import WebClient
-
-    _HAVE_SLACK_SDK = True
 except ImportError:
-    _HAVE_SLACK_SDK = False
+    raise ModuleNotFoundError("csp's slack adapter requires `slack-sdk`")
 
 T = TypeVar("T")
 log = getLogger(__file__)
@@ -55,8 +53,6 @@ def mention_user(userid: str) -> str:
 
 class SlackAdapterManager(AdapterManagerImpl):
     def __init__(self, app_token: str, bot_token: str, ssl: Optional[SSLContext] = None):
-        if not _HAVE_SLACK_SDK:
-            raise RuntimeError("Could not find slack-sdk installation")
         if not app_token.startswith("xapp-") or not bot_token.startswith("xoxb-"):
             raise RuntimeError("Slack app token or bot token looks malformed")
 

csp/adapters/symphony.py

@@ -1,6 +1,5 @@
 import http.client
 import json
-import requests
 import ssl
 import threading
 from logging import getLogger
@@ -14,6 +13,11 @@
 from csp.impl.pushadapter import PushInputAdapter
 from csp.impl.wiring import py_push_adapter_def
 
+try:
+    import requests
+except ImportError:
+    raise ModuleNotFoundError("csp's symphony adapter requires `requests`")
+
 __all__ = ["SymphonyAdapter", "SymphonyMessage"]
 
 log = getLogger(__file__)

csp/adapters/websocket.py

@@ -20,7 +20,13 @@
 )
 from csp.impl.wiring import input_adapter_def, output_adapter_def, status_adapter_def
 from csp.impl.wiring.delayed_node import DelayedNodeWrapperDef
-from csp.lib import _websocketadapterimpl
+
+try:
+    from csp.lib import _websocketadapterimpl
+except ImportError:
+    raise ImportError(
+        "csp's websocket adapter requires the C++ csp extension to be built, but it could not be imported"
+    )
 
 from .websocket_types import WebsocketHeaderUpdate
 
@@ -39,14 +45,14 @@
     import tornado.web
     import tornado.websocket
 except ImportError:
-    raise ImportError("websocket adapter requires tornado package")
+    raise ModuleNotFoundError("csp's websocket adapter requires `tornado`")
 
 try:
     import rapidjson
 
     datetime_mode = rapidjson.DM_UNIX_TIME | rapidjson.DM_NAIVE_IS_UTC
 except ImportError:
-    raise ImportError("websocket adapter requires rapidjson package")
+    raise ModuleNotFoundError("csp's websocket adapter requires `rapidjson`")
 
 
 def diff_dict(old, new):

pyproject.toml

@@ -19,14 +19,12 @@ requires-python = ">=3.8"
 
 dependencies = [
     "backports.zoneinfo; python_version<'3.9'",
-    "numpy",
+    "numpy>=1.19",
     "packaging",
-    "pandas",
     "psutil",
     "pyarrow>=7.0.0",
     "pytz",
     "ruamel.yaml",
-    "sqlalchemy",
 ]
 
 classifiers = [
@@ -75,6 +73,7 @@ develop = [
     "pillow",
     # adapters
     "httpx>=0.20,<1",  # kafka
+    "pandas",  # pandas extension
     "polars",  # parquet
     "psutil",  # test_engine/test_history
     "slack-sdk>=3",  # slack