diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index 711b2c883..99d565659 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -126,6 +126,7 @@ def _source_name(self): def _parse_header(self): self.signals_info_list = scan_files(self.dirname) + _add_segment_order(self.signals_info_list) # sort stream_name by higher sampling rate first srates = {info["stream_name"]: info["sampling_rate"] for info in self.signals_info_list} @@ -165,8 +166,9 @@ def _parse_header(self): sync_stream_id_to_buffer_id = {} for stream_name in stream_names: - # take first segment - info = self.signals_info_dict[0, stream_name] + # take first segment to extract the signal info + segment_index = 0 + info = self.signals_info_dict[segment_index, stream_name] buffer_id = stream_name buffer_name = stream_name @@ -176,22 +178,19 @@ def _parse_header(self): signal_streams.append((stream_name, stream_id, buffer_id)) - # add channels to global list - for local_chan in range(info["num_chan"]): - chan_name = info["channel_names"][local_chan] + # add channels to signal channel header + for local_channel_index in range(info["num_chan"]): + chan_name = info["channel_names"][local_channel_index] chan_id = f"{stream_name}#{chan_name}" - # Sync channel - if ( - "nidq" not in stream_name - and "SY0" in chan_name - and not self.load_sync_channel - and local_chan == info["num_chan"] - 1 - ): + # Separate sync channel in its own stream + is_sync_channel = "SY" in chan_name and not self.load_sync_channel + if is_sync_channel : # This is a sync channel and should be added as its own stream sync_stream_id = f"{stream_name}-SYNC" sync_stream_id_to_buffer_id[sync_stream_id] = buffer_id stream_id_for_chan = sync_stream_id + else: stream_id_for_chan = stream_id @@ -202,17 +201,18 @@ def _parse_header(self): info["sampling_rate"], "int16", info["units"], - info["channel_gains"][local_chan], - info["channel_offsets"][local_chan], + info["channel_gains"][local_channel_index], + info["channel_offsets"][local_channel_index], stream_id_for_chan, buffer_id, ) ) - # all channel by default unless load_sync_channel=False + # None here means that the all the channels in the buffer will be included self._stream_buffer_slice[stream_id] = None - # check sync channel validity + # Then we modify this if sync channel is present to slice the last channel + # out of the stream buffer if "nidq" not in stream_name: if not self.load_sync_channel and info["has_sync_trace"]: # the last channel is removed from the stream but not from the buffer @@ -227,7 +227,7 @@ def _parse_header(self): signal_buffers = np.array(signal_buffers, dtype=_signal_buffer_dtype) - # Add sync channels as their own streams + # Add sync channels as their own streams. We do it here to keep the order of the streams for sync_stream_id, buffer_id in sync_stream_id_to_buffer_id.items(): signal_streams.append((sync_stream_id, sync_stream_id, buffer_id)) @@ -259,7 +259,6 @@ def _parse_header(self): spike_channels = np.array(spike_channels, dtype=_spike_channel_dtype) # deal with nb_segment and t_start/t_stop per segment - self._t_starts = {stream_name: {} for stream_name in stream_names} self._t_stops = {seg_index: 0.0 for seg_index in range(nb_segment)} @@ -378,12 +377,7 @@ def _get_analogsignal_buffer_description(self, block_index, seg_index, buffer_id def scan_files(dirname): """ - Scan for pairs of `.bin` and `.meta` files and return information about it. - - After exploring the folder, the segment index (`seg_index`) is construct as follow: - * if only one `gate_num=0` then `trigger_num` = `seg_index` - * if only one `trigger_num=0` then `gate_num` = `seg_index` - * if both are increasing then seg_index increased by gate_num, trigger_num order. + Scan for pairs of `.bin` and `.meta` files and parse the metadata file to extract signal information. """ info_list = [] @@ -393,6 +387,7 @@ def scan_files(dirname): continue meta_filename = Path(root) / file bin_filename = meta_filename.with_suffix(".bin") + if meta_filename.exists() and bin_filename.exists(): meta = read_meta_file(meta_filename) info = extract_stream_info(meta_filename, meta) @@ -404,9 +399,21 @@ def scan_files(dirname): if len(info_list) == 0: raise FileNotFoundError(f"No appropriate combination of .meta and .bin files were detected in {dirname}") + return info_list + +def _add_segment_order(info_list): + """ + Uses gate and trigger numbers to construct a segment index (`seg_index`) for each signal in `info_list`. + + After exploring the folder, the segment index (`seg_index`) is construct as follow: + * if only one `gate_num=0` then `trigger_num` = `seg_index` + * if only one `trigger_num=0` then `gate_num` = `seg_index` + * if both are increasing then seg_index increased by gate_num, trigger_num order. + + """ # This sets non-integers values before integers normalize = lambda x: x if isinstance(x, int) else -1 - + # Segment index is determined by the gate_num and trigger_num in that order def get_segment_tuple(info): # Create a key from the normalized gate_num and trigger_num @@ -423,25 +430,6 @@ def get_segment_tuple(info): for info in info_list: info["seg_index"] = segment_tuple_to_segment_index[get_segment_tuple(info)] - for info in info_list: - # device_kind is imec, nidq - if info.get("device_kind") == "imec": - info["device_index"] = info["device"].split("imec")[-1] - else: - info["device_index"] = "" # TODO: Handle multi nidq case, maybe use meta["typeNiEnabled"] - - # Define stream base on device_kind [imec|nidq], device_index and stream_kind [ap|lf] for imec - # Stream format is "{device_kind}{device_index}.{stream_kind}" - for info in info_list: - device_kind = info["device_kind"] - device_index = info["device_index"] - stream_kind = f".{info['stream_kind']}" if info["stream_kind"] else "" - stream_name = f"{device_kind}{device_index}{stream_kind}" - - info["stream_name"] = stream_name - - return info_list - def parse_spikeglx_fname(fname): """ @@ -451,13 +439,13 @@ def parse_spikeglx_fname(fname): https://github.com/billkarsh/SpikeGLX/blob/15ec8898e17829f9f08c226bf04f46281f106e5f/Markdown/UserManual.md#gates-and-triggers Example file name structure: - Consider the filenames: `Noise4Sam_g0_t0.nidq.bin` or `Noise4Sam_g0_t0.imec0.lf.bin` + Consider the filenames: `Noise4Sam_g0_t0.nidq.bin`, `Noise4Sam_g0_t0.imec0.lf.bin`, or `myRun_g0_t0.obx0.obx.bin` The filenames consist of 3 or 4 parts separated by `.` 1. "Noise4Sam_g0_t0" will be the `name` variable. This choosen by the user at recording time. 2. "g0" is the "gate_num" 3. "t0" is the "trigger_num" - 4. "nidq" or "imec0" will give the `device` - 5. "lf" or "ap" will be the `stream_kind` + 4. "nidq", "imec0", or "obx0" will give the `device` + 5. "lf", "ap", or "obx" will be the `stream_kind` `stream_name` variable is the concatenation of `device.stream_kind` If CatGT is used, then the trigger numbers in the file names ("t0"/"t1"/etc.) @@ -472,7 +460,7 @@ def parse_spikeglx_fname(fname): Parameters --------- fname: str - The filename to parse without the extension, e.g. "my-run-name_g0_t1.imec2.lf" + The filename to parse without the extension, e.g. "my-run-name_g0_t1.imec2.lf" or "my-run-name_g0_t0.obx0.obx" Returns ------- @@ -483,45 +471,86 @@ def parse_spikeglx_fname(fname): trigger_num: int | str or None The trigger identifier, e.g. 1. If CatGT is used, then the trigger_num will be set to "cat". device: str - The probe identifier, e.g. "imec2" + The probe identifier, e.g. "imec2" or "obx0" stream_kind: str or None - The data type identifier, "lf" or "ap" or None + The data type identifier, "lf", "ap", "obx", or None """ - re_standard = re.findall(r"(\S*)_g(\d*)_t(\d*)\.(\S*).(ap|lf)", fname) - re_tcat = re.findall(r"(\S*)_g(\d*)_tcat.(\S*).(ap|lf)", fname) - re_nidq = re.findall(r"(\S*)_g(\d*)_t(\d*)\.(\S*)", fname) - if len(re_standard) == 1: - # standard case with probe - run_name, gate_num, trigger_num, device, stream_kind = re_standard[0] - elif len(re_tcat) == 1: - # tcat case - run_name, gate_num, device, stream_kind = re_tcat[0] - trigger_num = "cat" - elif len(re_nidq) == 1: - # case for nidaq - run_name, gate_num, trigger_num, device = re_nidq[0] - stream_kind = None - else: - # the naming do not correspond lets try something more easy - # example: sglx_xxx.imec0.ap - re_else = re.findall(r"(\S*)\.(\S*).(ap|lf)", fname) - re_else_nidq = re.findall(r"(\S*)\.(\S*)", fname) - if len(re_else) == 1: - run_name, device, stream_kind = re_else[0] - gate_num, trigger_num = None, None - elif len(re_else_nidq) == 1: - # easy case for nidaq, example: sglx_xxx.nidq - run_name, device = re_else_nidq[0] - gate_num, trigger_num, stream_kind = None, None, None - else: - raise ValueError(f"Cannot parse filename {fname}") - - if gate_num is not None: - gate_num = int(gate_num) - if trigger_num is not None and trigger_num != "cat": - trigger_num = int(trigger_num) - - return (run_name, gate_num, trigger_num, device, stream_kind) + + # Standard case: contains gate, trigger, device, and stream kind + # Example: Noise4Sam_g0_t0.imec0.ap + # Format: {run_name}_g{gate_num}_t{trigger_num}.{device}.{stream_kind} + # Regex tokens: + # \S+ → one or more non-whitespace characters + # \d+ → one or more digits + # ap|lf → either 'ap' or 'lf' + regex = r"(?P\S+)_g(?P\d+)_t(?P\d+)\.(?P\S+)\.(?Pap|lf)" + match = re.match(regex, fname) + if match: + gd = match.groupdict() + return gd["run_name"], int(gd["gate_num"]), int(gd["trigger_num"]), gd["device"], gd["stream_kind"] + + # CatGT case: trigger renamed to 'tcat' + # Example: Noise4Sam_g0_tcat.imec0.ap + # Format: {run_name}_g{gate_num}_tcat.{device}.{stream_kind} + # Regex tokens: + # \S+ → one or more non-whitespace characters + # \d+ → one or more digits + # ap|lf → either 'ap' or 'lf' + regex = r"(?P\S+)_g(?P\d+)_tcat\.(?P\S+)\.(?Pap|lf)" + match = re.match(regex, fname) + if match: + gd = match.groupdict() + return gd["run_name"], int(gd["gate_num"]), "cat", gd["device"], gd["stream_kind"] + + # OneBox case: ends in .obx + # Example: myRun_g0_t0.obx0.obx + # Format: {run_name}_g{gate_num}_t{trigger_num}.{device}.obx + # Regex tokens: + # \S+ → one or more non-whitespace characters + # \d+ → one or more digits + regex = r"(?P\S+)_g(?P\d+)_t(?P\d+)\.(?P\S+)\.obx" + match = re.match(regex, fname) + if match: + gd = match.groupdict() + return gd["run_name"], int(gd["gate_num"]), int(gd["trigger_num"]), gd["device"], "obx" + + # NIDQ case no stream kind (not ap or lf) + # Example: Noise4Sam_g0_t0.nidq + # Format: {run_name}_g{gate_num}_t{trigger_num}.{device} + # Regex tokens: + # \S+ → one or more non-whitespace characters + # \d+ → one or more digits + regex = r"(?P\S+)_g(?P\d+)_t(?P\d+)\.(?P\S+)" + match = re.match(regex, fname) + if match: + gd = match.groupdict() + return gd["run_name"], int(gd["gate_num"]), int(gd["trigger_num"]), gd["device"], None + + # Fallback case 1): no gate/trigger, includes device and stream kind + # Example: sglx_name.imec0.ap + # Format: {run_name}.{device}.{stream_kind} + # Regex tokens: + # \S+ → one or more non-whitespace characters + # ap|lf → either 'ap' or 'lf' + regex = r"(?P\S+)\.(?P\S+)\.(?Pap|lf)" + match = re.match(regex, fname) + if match: + gd = match.groupdict() + return gd["run_name"], None, None, gd["device"], gd["stream_kind"] + + # Fallback NIDQ-style: no stream kind + # Example: sglx_name.nidq + # Format: {run_name}.{device} + # Regex tokens: + # \S+ → one or more non-whitespace characters + regex = r"(?P\S+)\.(?P\S+)" + match = re.match(regex, fname) + if match: + gd = match.groupdict() + return gd["run_name"], None, None, gd["device"], None + + # No known pattern matched + raise ValueError(f"Cannot parse filename {fname}") def read_meta_file(meta_file): @@ -554,13 +583,17 @@ def extract_stream_info(meta_file, meta): # AP and LF meta have this field ap, lf, sy = [int(s) for s in meta["snsApLfSy"].split(",")] has_sync_trace = sy == 1 + elif "snsXaDwSy" in meta: + # OneBox case + xa, dw, sy = [int(s) for s in meta["snsXaDwSy"].split(",")] + has_sync_trace = sy == 1 else: # NIDQ case has_sync_trace = False # This is the original name that the file had. It might not match the current name if the user changed it - bin_file_path = meta["fileName"] - fname = Path(bin_file_path).stem + original_bin_file_path = meta["fileName"] + fname = Path(original_bin_file_path).stem run_name, gate_num, trigger_num, device, stream_kind = parse_spikeglx_fname(fname) @@ -603,6 +636,18 @@ def extract_stream_info(meta_file, meta): channel_gains = gain_factor * per_channel_gain * 1e6 else: raise NotImplementedError("This meta file version of spikeglx" " is not implemented") + elif meta.get("typeThis") == "obx": + # OneBox case + device = fname.split(".")[-2] if "." in fname else device + stream_kind = "" + units = "V" + + # OneBox gain calculation + # V = i * Vmax / Imax where Imax = obMaxInt, Vmax = obAiRangeMax + # See: https://billkarsh.github.io/SpikeGLX/Sgl_help/Metadata_30.html + max_int = int(meta["obMaxInt"]) + gain_factor = float(meta["obAiRangeMax"]) / max_int + channel_gains = np.ones(num_chan) * gain_factor else: device = fname.split(".")[-1] stream_kind = "" @@ -624,11 +669,14 @@ def extract_stream_info(meta_file, meta): probe_slot = meta.get("imDatPrb_slot", None) probe_port = meta.get("imDatPrb_port", None) probe_dock = meta.get("imDatPrb_dock", None) + + # OneBox specific metadata + obx_slot = meta.get("imDatObx_slot", None) info = {} info["fname"] = fname info["meta"] = meta - for k in ("niSampRate", "imSampRate"): + for k in ("niSampRate", "imSampRate", "obSampRate"): if k in meta: info["sampling_rate"] = float(meta[k]) info["num_chan"] = num_chan @@ -648,8 +696,26 @@ def extract_stream_info(meta_file, meta): info["probe_slot"] = int(probe_slot) if probe_slot else None info["probe_port"] = int(probe_port) if probe_port else None info["probe_dock"] = int(probe_dock) if probe_dock else None + info["obx_slot"] = int(obx_slot) if obx_slot else None + + # Add device index + if info.get("device_kind") == "imec": + info["device_index"] = info["device"].split("imec")[-1] + elif info.get("device_kind") == "obx": + info["device_index"] = info["device"].split("obx")[-1] + else: + info["device_index"] = "" # TODO: Handle multi nidq case, maybe use meta["typeNiEnabled"] - if "nidq" in device: + # Define stream base on device_kind [imec|nidq|obx], device_index and stream_kind [ap|lf] for imec + # Stream format is "{device_kind}{device_index}.{stream_kind}" + device_kind = info["device_kind"] + device_index = info["device_index"] + stream_kind = f".{info['stream_kind']}" if info["stream_kind"] else "" + info["stream_name"] = f"{device_kind}{device_index}{stream_kind}" + + # Separate analog and digital channels + if device_kind == "nidq": + # Note that we only handling the non-multiplexed channels here info["digital_channels"] = [] info["analog_channels"] = [channel for channel in info["channel_names"] if not channel.startswith("XD")] # Digital/event channels are encoded within the digital word, so that will need more handling @@ -661,4 +727,10 @@ def extract_stream_info(meta_file, meta): info["digital_channels"].extend([f"XD{i}" for i in range(start, end + 1)]) else: info["digital_channels"].append(f"XD{int(item)}") + elif device_kind == "obx": + # OneBox channel handling - focus on analog channels only as requested + info["digital_channels"] = [channel for channel in info["channel_names"] if channel.startswith("XD")] + info["analog_channels"] = [channel for channel in info["channel_names"] if channel.startswith("XA")] + + return info diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py index 2da1a96b0..409b324d3 100644 --- a/neo/test/rawiotest/test_spikeglxrawio.py +++ b/neo/test/rawiotest/test_spikeglxrawio.py @@ -41,6 +41,8 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase): "spikeglx/multi_trigger_multi_gate/CatGT/CatGT-D", "spikeglx/multi_trigger_multi_gate/CatGT/CatGT-E", "spikeglx/multi_trigger_multi_gate/CatGT/Supercat-A", + # One Box" + "spikeglx/onebox/run_with_only_adc", ] def test_loading_only_one_probe_in_multi_probe_scenario(self):