Merge pull request #1932 from aaronkanzer/fix/bci2000-scaling

Fix BCI2000RawIO scaling: decimal gains were parsed as 0 and the offset was applied with the wrong convention

Author: h-mayorquin

neo/rawio/bci2000rawio.py

@@ -24,6 +24,17 @@
 )
 from neo.core import NeoReadWriteError
 
+# A decimal number followed by an optional unit
+# Example: "0.01617", "-1.5e-3", "0.003muV", "256.0Hz"
+# Format:  {number}{unit}
+# Regex tokens:
+#   [-+]?           → optional sign
+#   \d+\.?\d*       → digits with an optional decimal part, e.g. "43", "5.", "0.01617"
+#   \.\d+           → or a number that starts at the decimal point, e.g. ".5"
+#   [eE][-+]?\d+    → optional exponent, e.g. "e-3"
+#   \w*             → the unit, e.g. "muV", possibly empty
+_number_pattern = r"(?P<number>[-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)(?P<unit>\w*)"
+
 
 class BCI2000RawIO(BaseRawIO):
     """
@@ -74,16 +85,16 @@ def _parse_header(self):
             gain = param_defs["SourceChGain"]["value"][chan_ix]
 
             if isinstance(gain, str):
-                r = re.findall(r"(\d+)(\D+)", gain)
                 # some files have strange units attached to gain
                 # in that case it is ignored
-                if len(r) == 1:
-                    gain = r[0][0]
-                gain = float(gain)
+                gain = float(re.match(_number_pattern, gain).group("number"))
 
             offset = param_defs["SourceChOffset"]["value"][chan_ix]
             if isinstance(offset, str):
-                offset = float(offset)
+                offset = float(re.match(_number_pattern, offset).group("number"))
+            # BCI2000 defines uV = (raw - SourceChOffset) * SourceChGain
+            # while neo computes raw * gain + offset
+            offset = -offset * gain
 
             sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id, buffer_id))
         self.header["signal_channels"] = np.array(sig_channels, dtype=_signal_channel_dtype)
@@ -280,7 +291,8 @@ def rescale_value(param_val, data_type):
         if param_val.lower().startswith("0x"):
             param_val = int(param_val, 16)
         elif data_type in ["int", "float"]:
-            matches = re.match(r"(-*\d+)(\w*)", param_val)
+            pattern = r"(-*\d+)(\w*)" if data_type == "int" else _number_pattern
+            matches = re.match(pattern, param_val)
             if matches is not None:  # Can be None for % in def, min, max vals
                 param_val, unit_str = matches.group(1), matches.group(2)
                 param_val = int(param_val) if data_type == "int" else float(param_val)
@@ -325,7 +337,9 @@ def parse_dimensions(param_list):
         if temp != "[ State Vector Definition ]":
             raise NeoReadWriteError(f"State definitions not found in header {filename}")
         state_defs = []
-        state_def_dtype = [("name", "a64"), ("length", int), ("startVal", int), ("bytePos", int), ("bitPos", int)]
+        # "S64" because NumPy 2.0 deprecated the "a64" alias (https://github.com/numpy/numpy/pull/24854)
+        # and NumPy 2.5 removed it, see https://github.com/NeuralEnsemble/python-neo/pull/1932
+        state_def_dtype = [("name", "S64"), ("length", int), ("startVal", int), ("bytePos", int), ("bitPos", int)]
         while True:
             temp = fid.readline().decode("utf8").strip()
             if len(temp) == 0 or temp[0] == "[":

neo/test/rawiotest/test_bci2000rawio.py

@@ -4,6 +4,8 @@
 
 import unittest
 
+import numpy as np
+
 from neo.rawio.bci2000rawio import BCI2000RawIO
 from neo.test.rawiotest.common_rawio_test import BaseTestRawIO
 
@@ -14,12 +16,20 @@ class TestBCI2000RawIO(
 ):
     rawioclass = BCI2000RawIO
 
-    entities_to_download = ["bci2000/eeg1_1.dat", "bci2000/eeg1_2.dat", "bci2000/eeg1_3.dat"]
-
     entities_to_download = [
         "bci2000",
     ]
 
+    entities_to_test = ["bci2000/eeg1_1.dat", "bci2000/eeg1_2.dat", "bci2000/eeg1_3.dat"]
+
+    def test_scaling(self):
+        reader = BCI2000RawIO(filename=self.get_local_path("bci2000/eeg1_1.dat"))
+        reader.parse_header()
+        raw = reader.get_analogsignal_chunk(0, 0, 0, 3, 0, channel_indexes=[0])
+        uV = reader.rescale_signal_raw_to_float(raw, dtype="float64", stream_index=0, channel_indexes=[0])
+        # first samples as decoded by BCI2000's own FileReader.py (BCI2kReader)
+        np.testing.assert_allclose(uV[:, 0], [-16.218512, 1.3744501, -9.23307], rtol=1e-5)
+
 
 if __name__ == "__main__":
     unittest.main()