feature: function to verify that keys from old splits have not moved

Author: FelixAbrahamsson

datastream/tools/__init__.py

@@ -5,3 +5,4 @@
 from datastream.tools.split_dataframes import split_dataframes
 from datastream.tools.unstratified_split import unstratified_split
 from datastream.tools.stratified_split import stratified_split
+from datastream.tools.verify_split import verify_split

datastream/tools/verify_split.py

@@ -0,0 +1,33 @@
+import json
+from pathlib import Path
+from pydantic import validate_arguments
+
+
+@validate_arguments
+def verify_split(old_path: Path, new_path: Path):
+    '''
+    Verify that no keys from an old split are present in a different new split.
+
+    .. highlight:: python
+    .. code-block:: python
+
+        verify_split(
+            "path/to/old/split.json",
+            "path/to/new/split.json",
+        )
+
+    '''
+    for old_split_name, old_split in json.loads(old_path.read_text()).items():
+        for new_split_name, new_split in json.loads(new_path.read_text()).items():
+            if (
+                old_split_name != new_split_name
+                and len(set(old_split).intersection(set(new_split))) > 0
+            ):
+                raise ValueError(
+                    f'Some keys from old split "{old_split_name}"'
+                    f' are present in new split "{new_split_name}":\n'
+                    + str("\n".join(
+                        [str(old_split[index]) for index in range(min(10, len(old_split)))]
+                        + (["..."] if len(old_split) > 10 else [])
+                    ))
+                )

docs/source/datastream.rst

@@ -1,6 +1,7 @@
 
 Datastream
 =====================
+
 .. autoclass:: datastream.Datastream
    :members:
    :member-order: bysource

docs/source/index.rst

@@ -20,6 +20,7 @@ sampling, and finally converting to a ``torch.utils.data.DataLoader``.
    get_started
    dataset
    datastream
+   tools
 
 Indices and tables
 ==================

docs/source/requirements.txt

@@ -9,7 +9,7 @@ certifi==2020.4.5.2
 cffi==1.14.0
 chardet==3.0.4
 commonmark==0.9.1
-cryptography>=3.2.0
+cryptography==3.4.7
 decorator==4.4.2
 docutils==0.16
 idna==2.9
@@ -30,10 +30,11 @@ pkginfo==1.5.0.1
 pluggy==0.13.1
 py==1.10.0
 pycparser==2.20
-pydantic==1.5.1
+pydantic==1.8.1
 Pygments==2.7.4
 pylint==2.5.3
 pyparsing==2.4.7
+pyspark==2.4.1
 pytest==5.4.3
 python-dateutil==2.8.1
 pytz==2020.1
@@ -46,7 +47,7 @@ SecretStorage==3.1.2
 six==1.15.0
 snowballstemmer==2.0.0
 soupsieve==2.0.1
-Sphinx==3.1.0
+Sphinx==3.5.4
 sphinx-jsonschema==1.15
 sphinx-pydantic==0.1.1
 sphinx-rtd-theme==0.4.3
@@ -57,9 +58,10 @@ sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.4
 toml==0.10.1
-torch==1.4.0
+torch==1.8.1
 tqdm==4.46.1
 twine==3.1.1
+typing-extensions==3.10.0.0
 urllib3==1.25.9
 waitress==1.4.4
 wcwidth==0.2.4

docs/source/tools.rst

@@ -0,0 +1,5 @@
+
+tools
+=====================
+
+.. autofunction:: datastream.tools.verify_split