From a285a75dda97ec98259a407e2831c42f7270396a Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 29 Sep 2026 15:28:23 -0600 Subject: [PATCH 1/4] draft --- neo/rawio/neuroexplorerrawio.py | 41 ++++++++++-- neo/test/rawiotest/test_neuroexplorerrawio.py | 62 ++++++++++++++++++- 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/neo/rawio/neuroexplorerrawio.py b/neo/rawio/neuroexplorerrawio.py index 9cda296e3..3ce48c0db 100644 --- a/neo/rawio/neuroexplorerrawio.py +++ b/neo/rawio/neuroexplorerrawio.py @@ -60,6 +60,13 @@ def _parse_header(self): self._memmap = np.memmap(self.filename, dtype="u1", mode="r") + # The file version does not change the header layout, it declares which of the fixed + # fields carry a meaningful value. NexFile.h: versions 100, 101 and 104 leave both + # MVOffset and PrethresholdTimeInSeconds at zero, 105 is where MVOffset can be non-zero + # and 106 where PrethresholdTimeInSeconds can be. + file_version = self.global_header["version"] + mv_offset_is_valid = file_version >= 105 + self._sig_lengths = [] self._sig_t_starts = [] sig_channels = [] @@ -81,9 +88,16 @@ def _parse_header(self): elif entity_header["type"] == 3: # spiketrain and waveforms wf_units = "mV" wf_gain = entity_header["ADtoMV"] - wf_offset = entity_header["MVOffset"] - wf_left_sweep = 0 + wf_offset = entity_header["MVOffset"] if mv_offset_is_valid else 0.0 wf_sampling_rate = entity_header["WFrequency"] + # PrethresholdTimeInSeconds is the specification's own definition of neo's left + # sweep: if a waveform timestamp is t, its first point is at t - Prethreshold. + # neo counts the left sweep in samples. The variable's own version gates the + # field independently of the file version, 102 being where it is valid. + if file_version >= 106 and entity_header["varVersion"] >= 102: + wf_left_sweep = int(round(entity_header["PrethresholdTimeInSeconds"] * wf_sampling_rate)) + else: + wf_left_sweep = 0 spike_channels.append((name, _id, wf_units, wf_gain, wf_offset, wf_left_sweep, wf_sampling_rate)) elif entity_header["type"] == 4: @@ -95,7 +109,7 @@ def _parse_header(self): sampling_rate = entity_header["WFrequency"] dtype = "int16" gain = entity_header["ADtoMV"] - offset = entity_header["MVOffset"] + offset = entity_header["MVOffset"] if mv_offset_is_valid else 0.0 stream_id = str(_id) buffer_id = "" sig_channels.append((name, _id, sampling_rate, dtype, units, gain, offset, stream_id, buffer_id)) @@ -140,6 +154,24 @@ def _parse_header(self): d["neuroexplorer_version"] = self.global_header["version"] d["comment"] = self.global_header["comment"] + # Per-variable metadata, surfaced only where the specification says it is meaningful. + # WireNumber and UnitNumber are the channel and unit of the source record, so they are + # the electrode and unit identity a conversion needs; they apply to neurons and + # waveforms and the variable's own version gates them at 101. Gain, Filter, XPos and + # YPos apply to neurons alone, XPos and YPos being electrode positions in a (0, 100) + # range, so a waveform variable would report a position of (0, 0) that is not its own. + for spike_channel_index, spike_channel in enumerate(spike_channels): + entity_header = self._entity_headers[int(spike_channel["id"])] + spike_annotations = seg_annotations["spikes"][spike_channel_index] + if entity_header["varVersion"] >= 101: + spike_annotations["wire_number"] = int(entity_header["WireNumber"]) + spike_annotations["unit_number"] = int(entity_header["UnitNumber"]) + if entity_header["type"] == 0: # Unit + spike_annotations["gain"] = int(entity_header["Gain"]) + spike_annotations["filter"] = int(entity_header["Filter"]) + spike_annotations["x_pos"] = float(entity_header["XPos"]) + spike_annotations["y_pos"] = float(entity_header["YPos"]) + def _segment_t_start(self, block_index, seg_index): t_start = self.global_header["tbeg"] / self.global_header["freq"] return t_start @@ -324,7 +356,8 @@ def read_as_dict(fid, dtype, offset=None): ("NMarkers", "int32"), ("MarkerLength", "int32"), ("MVOffset", "float64"), - ("dummy", "S60"), + ("PrethresholdTimeInSeconds", "float64"), + ("dummy", "S52"), ] MarkerHeader = [ diff --git a/neo/test/rawiotest/test_neuroexplorerrawio.py b/neo/test/rawiotest/test_neuroexplorerrawio.py index 3f5f1223c..10fba8dbf 100644 --- a/neo/test/rawiotest/test_neuroexplorerrawio.py +++ b/neo/test/rawiotest/test_neuroexplorerrawio.py @@ -14,9 +14,11 @@ class TestNeuroExplorerRawIO( ): rawioclass = NeuroExplorerRawIO entities_to_download = ["neuroexplorer"] - files_to_download = [ + entities_to_test = [ "neuroexplorer/File_neuroexplorer_1.nex", "neuroexplorer/File_neuroexplorer_2.nex", + "neuroexplorer/nex_version_106/nonzero_prethreshold.nex", + "neuroexplorer/nex_version_106/nonzero_mv_offset.nex", ] def test_spike_waveforms(self): @@ -57,6 +59,64 @@ def test_data_offset_above_two_gigabytes(self): parsed = np.frombuffer(bytes(buffer), dtype=entity_dtype)[0] assert parsed["offset"] == true_offset + def test_version_106_prethreshold(self): + """A nonzero PrethresholdTimeInSeconds used to make the file unopenable. + + The variable header ends with that double followed by 52 bytes of padding, but the + reader declared the last 60 bytes as a single string field and decodes string fields + as UTF-8, so a nonzero double raised UnicodeDecodeError before any data was read. + The value is the left sweep, which neo counts in samples. + """ + assert np.dtype(EntityHeader).itemsize == 208 + + filename = self.get_local_path("neuroexplorer/nex_version_106/nonzero_prethreshold.nex") + reader = NeuroExplorerRawIO(filename=filename) + reader.parse_header() + + waveform_channels = [channel for channel in reader.header["spike_channels"] if channel["wf_sampling_rate"] > 0] + assert len(waveform_channels) == 6 + for channel in waveform_channels: + # 0.0002 s of prethreshold time at 40 kHz, so 8 of the 32 points precede the crossing + assert channel["wf_left_sweep"] == 8 + + def test_field_validity_follows_the_file_version(self): + """The file version declares which fixed header fields carry a meaningful value. + + PrethresholdTimeInSeconds is only meaningful from version 106 and MVOffset from 105, + so a version 104 file must report a left sweep of zero whatever its bytes hold. + """ + filename = self.get_local_path("neuroexplorer/File_neuroexplorer_2.nex") + reader = NeuroExplorerRawIO(filename=filename) + reader.parse_header() + assert reader.global_header["version"] == 104 + + names = [channel["name"] for channel in reader.header["spike_channels"]] + assert reader.header["spike_channels"][names.index("sig01i_wf")]["wf_left_sweep"] == 0 + + filename = self.get_local_path("neuroexplorer/nex_version_106/nonzero_mv_offset.nex") + reader = NeuroExplorerRawIO(filename=filename) + reader.parse_header() + assert reader.global_header["version"] == 106 + assert reader.header["signal_channels"][0]["offset"] == 0.125 + + def test_variable_metadata_annotations(self): + """WireNumber, UnitNumber and the electrode positions were parsed and never surfaced.""" + filename = self.get_local_path("neuroexplorer/nex_version_106/nonzero_prethreshold.nex") + reader = NeuroExplorerRawIO(filename=filename) + reader.parse_header() + + annotations = reader.raw_annotations["blocks"][0]["segments"][0]["spikes"] + neuron = annotations[0] + assert neuron["name"] == "Neuron04a" + assert neuron["wire_number"] == 0 + assert neuron["unit_number"] == 0 + assert (neuron["x_pos"], neuron["y_pos"]) == (16.67, 25.0) + + # Positions are documented for neurons only, so a waveform variable reports none + waveform = annotations[6] + assert waveform["name"] == "sig001a_wf" + assert "x_pos" not in waveform + if __name__ == "__main__": unittest.main() From bf2642679ad48a2a95b5848b7dd404fbb91362c2 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 2 Oct 2026 12:22:24 -0600 Subject: [PATCH 2/4] Keep the version 106 fix to the prethreshold field --- neo/rawio/neuroexplorerrawio.py | 28 ++-------------- neo/test/rawiotest/test_neuroexplorerrawio.py | 32 +++---------------- 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/neo/rawio/neuroexplorerrawio.py b/neo/rawio/neuroexplorerrawio.py index 3ce48c0db..8180d9440 100644 --- a/neo/rawio/neuroexplorerrawio.py +++ b/neo/rawio/neuroexplorerrawio.py @@ -60,12 +60,8 @@ def _parse_header(self): self._memmap = np.memmap(self.filename, dtype="u1", mode="r") - # The file version does not change the header layout, it declares which of the fixed - # fields carry a meaningful value. NexFile.h: versions 100, 101 and 104 leave both - # MVOffset and PrethresholdTimeInSeconds at zero, 105 is where MVOffset can be non-zero - # and 106 where PrethresholdTimeInSeconds can be. + # NexFile.h: PrethresholdTimeInSeconds can only be non-zero from file version 106 on file_version = self.global_header["version"] - mv_offset_is_valid = file_version >= 105 self._sig_lengths = [] self._sig_t_starts = [] @@ -88,7 +84,7 @@ def _parse_header(self): elif entity_header["type"] == 3: # spiketrain and waveforms wf_units = "mV" wf_gain = entity_header["ADtoMV"] - wf_offset = entity_header["MVOffset"] if mv_offset_is_valid else 0.0 + wf_offset = entity_header["MVOffset"] wf_sampling_rate = entity_header["WFrequency"] # PrethresholdTimeInSeconds is the specification's own definition of neo's left # sweep: if a waveform timestamp is t, its first point is at t - Prethreshold. @@ -109,7 +105,7 @@ def _parse_header(self): sampling_rate = entity_header["WFrequency"] dtype = "int16" gain = entity_header["ADtoMV"] - offset = entity_header["MVOffset"] if mv_offset_is_valid else 0.0 + offset = entity_header["MVOffset"] stream_id = str(_id) buffer_id = "" sig_channels.append((name, _id, sampling_rate, dtype, units, gain, offset, stream_id, buffer_id)) @@ -154,24 +150,6 @@ def _parse_header(self): d["neuroexplorer_version"] = self.global_header["version"] d["comment"] = self.global_header["comment"] - # Per-variable metadata, surfaced only where the specification says it is meaningful. - # WireNumber and UnitNumber are the channel and unit of the source record, so they are - # the electrode and unit identity a conversion needs; they apply to neurons and - # waveforms and the variable's own version gates them at 101. Gain, Filter, XPos and - # YPos apply to neurons alone, XPos and YPos being electrode positions in a (0, 100) - # range, so a waveform variable would report a position of (0, 0) that is not its own. - for spike_channel_index, spike_channel in enumerate(spike_channels): - entity_header = self._entity_headers[int(spike_channel["id"])] - spike_annotations = seg_annotations["spikes"][spike_channel_index] - if entity_header["varVersion"] >= 101: - spike_annotations["wire_number"] = int(entity_header["WireNumber"]) - spike_annotations["unit_number"] = int(entity_header["UnitNumber"]) - if entity_header["type"] == 0: # Unit - spike_annotations["gain"] = int(entity_header["Gain"]) - spike_annotations["filter"] = int(entity_header["Filter"]) - spike_annotations["x_pos"] = float(entity_header["XPos"]) - spike_annotations["y_pos"] = float(entity_header["YPos"]) - def _segment_t_start(self, block_index, seg_index): t_start = self.global_header["tbeg"] / self.global_header["freq"] return t_start diff --git a/neo/test/rawiotest/test_neuroexplorerrawio.py b/neo/test/rawiotest/test_neuroexplorerrawio.py index 10fba8dbf..4917654e3 100644 --- a/neo/test/rawiotest/test_neuroexplorerrawio.py +++ b/neo/test/rawiotest/test_neuroexplorerrawio.py @@ -79,11 +79,11 @@ def test_version_106_prethreshold(self): # 0.0002 s of prethreshold time at 40 kHz, so 8 of the 32 points precede the crossing assert channel["wf_left_sweep"] == 8 - def test_field_validity_follows_the_file_version(self): - """The file version declares which fixed header fields carry a meaningful value. + def test_prethreshold_ignored_before_version_106(self): + """PrethresholdTimeInSeconds is only meaningful from file version 106. - PrethresholdTimeInSeconds is only meaningful from version 106 and MVOffset from 105, - so a version 104 file must report a left sweep of zero whatever its bytes hold. + Before that the bytes are padding, so a version 104 file must report a left sweep of + zero whatever they hold. """ filename = self.get_local_path("neuroexplorer/File_neuroexplorer_2.nex") reader = NeuroExplorerRawIO(filename=filename) @@ -93,30 +93,6 @@ def test_field_validity_follows_the_file_version(self): names = [channel["name"] for channel in reader.header["spike_channels"]] assert reader.header["spike_channels"][names.index("sig01i_wf")]["wf_left_sweep"] == 0 - filename = self.get_local_path("neuroexplorer/nex_version_106/nonzero_mv_offset.nex") - reader = NeuroExplorerRawIO(filename=filename) - reader.parse_header() - assert reader.global_header["version"] == 106 - assert reader.header["signal_channels"][0]["offset"] == 0.125 - - def test_variable_metadata_annotations(self): - """WireNumber, UnitNumber and the electrode positions were parsed and never surfaced.""" - filename = self.get_local_path("neuroexplorer/nex_version_106/nonzero_prethreshold.nex") - reader = NeuroExplorerRawIO(filename=filename) - reader.parse_header() - - annotations = reader.raw_annotations["blocks"][0]["segments"][0]["spikes"] - neuron = annotations[0] - assert neuron["name"] == "Neuron04a" - assert neuron["wire_number"] == 0 - assert neuron["unit_number"] == 0 - assert (neuron["x_pos"], neuron["y_pos"]) == (16.67, 25.0) - - # Positions are documented for neurons only, so a waveform variable reports none - waveform = annotations[6] - assert waveform["name"] == "sig001a_wf" - assert "x_pos" not in waveform - if __name__ == "__main__": unittest.main() From ace9daede0319fe15915d912ba83271f0a7fd4a8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 2 Oct 2026 12:30:02 -0600 Subject: [PATCH 3/4] Remove unnecessary test --- neo/test/rawiotest/test_neuroexplorerrawio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/neo/test/rawiotest/test_neuroexplorerrawio.py b/neo/test/rawiotest/test_neuroexplorerrawio.py index 4917654e3..5e7fbb457 100644 --- a/neo/test/rawiotest/test_neuroexplorerrawio.py +++ b/neo/test/rawiotest/test_neuroexplorerrawio.py @@ -18,7 +18,6 @@ class TestNeuroExplorerRawIO( "neuroexplorer/File_neuroexplorer_1.nex", "neuroexplorer/File_neuroexplorer_2.nex", "neuroexplorer/nex_version_106/nonzero_prethreshold.nex", - "neuroexplorer/nex_version_106/nonzero_mv_offset.nex", ] def test_spike_waveforms(self): From 447491b620cca29b6912924747f5e82bbc5d6b36 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Fri, 2 Oct 2026 12:46:11 -0600 Subject: [PATCH 4/4] clarify dummy variable --- neo/rawio/neuroexplorerrawio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neo/rawio/neuroexplorerrawio.py b/neo/rawio/neuroexplorerrawio.py index 8180d9440..050ad8359 100644 --- a/neo/rawio/neuroexplorerrawio.py +++ b/neo/rawio/neuroexplorerrawio.py @@ -335,6 +335,7 @@ def read_as_dict(fid, dtype, offset=None): ("MarkerLength", "int32"), ("MVOffset", "float64"), ("PrethresholdTimeInSeconds", "float64"), + # Unused padding that fills the variable header to its fixed 208 bytes ("dummy", "S52"), ]