From 7b8644e2987c9dec51af68bbfb3b0aa7110e1280 Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Thu, 1 Oct 2026 17:07:16 -0500 Subject: [PATCH 1/2] Treat a NaN of any float width as missing column_source.is_missing tested isinstance(value, float), which numpy's float32 and float16 do not satisfy (only float64 subclasses float), so a narrow-dtype NaN from a numpy-backed table was handed to validation as the text 'nan'. The test now accepts any numbers.Real. ndx-hed filtered these itself (its adoption plan Q3); any other numpy-backed caller had the gap. --- hed/models/column_source.py | 12 +++++++++--- tests/models/test_column_source.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/hed/models/column_source.py b/hed/models/column_source.py index 9f3453cb..a5d28971 100644 --- a/hed/models/column_source.py +++ b/hed/models/column_source.py @@ -12,6 +12,7 @@ from __future__ import annotations import math +import numbers from typing import Protocol, runtime_checkable # Cell texts that mean "no value here". None and float NaN are also missing. @@ -45,11 +46,16 @@ def distinct_values(values) -> dict[str, list[int]]: def is_missing(value) -> bool: - """Return True if a cell value stands for "no value": None, a float NaN, ``""`` or ``"n/a"``.""" + """Return True if a cell value stands for "no value": None, a NaN of any float width, ``""`` or ``"n/a"``. + + The NaN test accepts any real number rather than only ``float``: numpy's float32 and float16 do not + subclass Python's float (only float64 does), and a numpy-backed table (an NWB column, a pandas + frame read with a narrow dtype) hands those NaNs over as values. + """ if value is None: return True - if isinstance(value, float) and math.isnan(value): - return True + if isinstance(value, numbers.Real): + return math.isnan(value) return isinstance(value, str) and value in MISSING_VALUES diff --git a/tests/models/test_column_source.py b/tests/models/test_column_source.py index 10b8d38c..b3715493 100644 --- a/tests/models/test_column_source.py +++ b/tests/models/test_column_source.py @@ -1,9 +1,11 @@ import math import unittest +import numpy as np import pandas as pd from hed.models import ColumnSource, ListColumnSource, TabularInput, distinct_values +from hed.models.column_source import is_missing class TestDistinctValues(unittest.TestCase): @@ -15,6 +17,20 @@ def test_missing_values_are_dropped(self): values = [None, float("nan"), "", "n/a", "Red", math.nan, "n/a"] self.assertEqual(distinct_values(values), {"Red": [4]}) + def test_numpy_nan_of_any_width_is_missing(self): + """numpy float32 and float16 are not instances of float; their NaN is still a missing value.""" + for dtype in (np.float16, np.float32, np.float64): + values = np.array([1.5, np.nan, 1.5], dtype=dtype) + self.assertEqual(distinct_values(values), {"1.5": [0, 2]}, dtype) + self.assertTrue(is_missing(values[1]), dtype) + self.assertFalse(is_missing(values[0]), dtype) + self.assertFalse(is_missing(np.int32(0))) + self.assertFalse(is_missing(np.bool_(False))) + self.assertFalse(is_missing(0)) + self.assertTrue(is_missing(None)) + self.assertTrue(is_missing("n/a")) + self.assertFalse(is_missing("nan")) # the text nan is a value + def test_non_strings_become_text_and_bytes_are_decoded(self): values = [3, 3.5, b"Blue", 3] self.assertEqual(distinct_values(values), {"3": [0, 3], "3.5": [1], "Blue": [2]}) From 136b989d17a669f671a2d60626604dfee3e7a6b7 Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Fri, 2 Oct 2026 06:42:40 -0500 Subject: [PATCH 2/2] Detect NaN by self-inequality, not math.isnan Copilot review of PR #1441. numbers.Real includes arbitrary-size integers and Fraction, and math.isnan converts its argument to float, so a valid value such as 10**1000 raised OverflowError inside distinct_values. NaN is the one real value unequal to itself, and that test converts nothing. --- hed/models/column_source.py | 7 ++++--- tests/models/test_column_source.py | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/hed/models/column_source.py b/hed/models/column_source.py index a5d28971..0a2f7312 100644 --- a/hed/models/column_source.py +++ b/hed/models/column_source.py @@ -11,7 +11,6 @@ from __future__ import annotations -import math import numbers from typing import Protocol, runtime_checkable @@ -50,12 +49,14 @@ def is_missing(value) -> bool: The NaN test accepts any real number rather than only ``float``: numpy's float32 and float16 do not subclass Python's float (only float64 does), and a numpy-backed table (an NWB column, a pandas - frame read with a narrow dtype) hands those NaNs over as values. + frame read with a narrow dtype) hands those NaNs over as values. NaN is the one real value that is + not equal to itself, and that test needs no conversion to float, so an integer too large for a + float or a Fraction is simply a value. """ if value is None: return True if isinstance(value, numbers.Real): - return math.isnan(value) + return bool(value != value) return isinstance(value, str) and value in MISSING_VALUES diff --git a/tests/models/test_column_source.py b/tests/models/test_column_source.py index b3715493..a99f8e1b 100644 --- a/tests/models/test_column_source.py +++ b/tests/models/test_column_source.py @@ -1,5 +1,6 @@ import math import unittest +from fractions import Fraction import numpy as np import pandas as pd @@ -31,6 +32,13 @@ def test_numpy_nan_of_any_width_is_missing(self): self.assertTrue(is_missing("n/a")) self.assertFalse(is_missing("nan")) # the text nan is a value + def test_real_values_that_do_not_fit_a_float_are_values(self): + """An integer too large for a float, or a Fraction, is a value: the NaN test converts nothing.""" + huge = 10**1000 + self.assertFalse(is_missing(huge)) + self.assertFalse(is_missing(Fraction(1, 3))) + self.assertEqual(distinct_values([huge, Fraction(1, 3), huge]), {str(huge): [0, 2], "1/3": [1]}) + def test_non_strings_become_text_and_bytes_are_decoded(self): values = [3, 3.5, b"Blue", 3] self.assertEqual(distinct_values(values), {"3": [0, 3], "3.5": [1], "Blue": [2]})