diff --git a/CHANGELOG.md b/CHANGELOG.md index b83127d..d3582bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ All notable changes to the `cuvis` Python wrapper are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Entry wording follows the conventions in [CONTRIBUTING.md](CONTRIBUTING.md#changelog-conventions) - every API entry names the fully qualified symbol first, then states the change with one of the fixed predicates. -Versions are `MAJOR.MINOR.PATCH.TWEAK`. -`MAJOR.MINOR.PATCH` is the cuvis SDK release the wrapper targets; `TWEAK` counts wrapper-only revisions against that same SDK. +Versions are `WORLD.MAJOR.MINOR.PATCH`. +`WORLD.MAJOR.MINOR` is the cuvis SDK release the wrapper targets; `PATCH` counts wrapper-only revisions against that same SDK. See [CONTRIBUTING.md](CONTRIBUTING.md#version-scheme) for the full scheme. Entries for versions released before this file existed were reconstructed from the published PyPI artifacts and from an AST-level diff of the public `cuvis` API surface between the corresponding commits. @@ -15,6 +15,14 @@ Pre-releases (`b*`, `rc*`) are not listed. ### Added +- `cuvis.ReferenceType.WhiteSpectrum` - new enum member. +- `cuvis.ReferenceType.TargetSpectrum` - new enum member. +- `cuvis.ProcessingContext.set_reference` - new parameter `effective_bit_depth: int | None = None`. +- `cuvis.ProcessingContext.set_reference` - new parameter `integration_time: float = 0.0`. +- `cuvis.ProcessingContext.get_reference_spectrum` - new method. + Returns the white or target reference spectrum as an `ImageData`, or `None` when the slot is empty. + Target values are reflectance fractions (1.0 = 100 percent); the white spectrum additionally carries `effective_bit_depth` and `integration_time` attributes. +- `tests/` - behavior tests for the reference spectra: a 0.5 reflectivity target halves the reflectance cube, white spectrum and white measurement clear each other, four times the white counts quarter the cube, and both spectra survive a legacy `.cu3` save behind `.cu3sp` sidecar links and the stored cube is bit-identical after reload. - `CI` - `.github/workflows/ci.yml` runs the test suite and a lint job enforcing `ruff check` and `ruff format --check` on every pull request and on every push to `develop` and `main`. - `CI` - `.github/workflows/release.yml` is driven by `v*.*.*.*` tags: it validates the tag against `pyproject.toml` and against this file, builds, publishes to TestPyPI, and publishes to PyPI plus a GitHub Release after manual approval. - `CI` - `scripts/check_changelog.py` validates this file's structure (header format, allowed section names, descending versions) and the tag/version/changelog agreement at release time. @@ -36,6 +44,8 @@ Pre-releases (`b*`, `rc*`) are not listed. ### Changed - Whole tree reformatted with `ruff format`; no behaviour change. +- `cuvis.ProcessingContext.set_reference` - type changed from `(mesu: Measurement, refType: ReferenceType)` to `(data: Measurement | ImageData | tuple, refType: ReferenceType)`. + The spectrum reference types take spectrum data instead of a `Measurement`; the first parameter is renamed from `mesu` to `data`. - `README.md` - documents the version scheme, and lists Python 3.14 among the supported interpreters as `pyproject.toml` already did. - `prebuild.py` - writes `cuvis/git-hash.txt` instead of `git-hash.txt` at the repository root, so the file lands inside the package that declares it as package data. - `cuvis.General.init` - parameter `settings_path` type changed from `str` to `Union[str, Path, SdkSettings]`. diff --git a/cuvis/ProcessingContext.py b/cuvis/ProcessingContext.py index 568596f..5e3d2f2 100644 --- a/cuvis/ProcessingContext.py +++ b/cuvis/ProcessingContext.py @@ -3,6 +3,7 @@ from .FileWriteSettings import ProcessingArgs from .Measurement import Measurement from .SessionFile import SessionFile +from .cube_utils import ImageData from .cuvis_aux import SDKException from .cuvis_types import ReferenceType, ProcessingMode @@ -11,6 +12,35 @@ import dataclasses +import numpy as np + +_SPECTRUM_REFERENCES = (ReferenceType.WhiteSpectrum, ReferenceType.TargetSpectrum) + + +def _spectrum_arrays(data: ImageData | tuple) -> tuple[np.ndarray, np.ndarray]: + """(wavelengths_nm, values) as validated 1-d arrays out of an ImageData or a pair.""" + if isinstance(data, ImageData): + if data.wavelength is None: + raise ValueError("The spectrum ImageData carries no wavelengths.") + wavelengths = np.asarray(data.wavelength).reshape(-1) + values = np.asarray(data.array).reshape(-1) + else: + try: + wavelengths, values = data + except (TypeError, ValueError): + raise TypeError( + "A reference spectrum is an ImageData with wavelengths, or a" + " (wavelengths, values) pair of one-dimensional arrays." + ) + wavelengths = np.asarray(wavelengths).reshape(-1) + values = np.asarray(values).reshape(-1) + if wavelengths.size == 0 or wavelengths.size != values.size: + raise ValueError( + "A reference spectrum needs equally many wavelengths and values," + f" got {wavelengths.size} and {values.size}." + ) + return np.ascontiguousarray(wavelengths, dtype=np.float32), values + class ProcessingContext(object): def __init__( @@ -60,9 +90,75 @@ def apply(self, mesu: Measurement) -> Measurement: raise SDKException("Can only apply ProcessingContext to Measurement!") pass - def set_reference(self, mesu: Measurement, refType: ReferenceType) -> None: - if cuvis_il.status_ok != cuvis_il.cuvis_proc_cont_set_reference( - self._handle, mesu._handle, internal.__CuvisReferenceType__[refType] + def set_reference( + self, + data: Measurement | ImageData | tuple, + refType: ReferenceType, + *, + effective_bit_depth: int | None = None, + integration_time: float = 0.0, + ) -> None: + """Set a reference for processing. + + The classic reference types take a Measurement. The two spectrum types take an + ImageData carrying wavelengths, or a (wavelengths, values) pair of arrays, with + wavelengths in nanometres: + + - TargetSpectrum: reflectance values as fractions, 1.0 meaning 100 percent. + - WhiteSpectrum: raw sensor counts (uint16); effective_bit_depth (1 to 16) is + required, integration_time [ms] describes the recording. + """ + if refType not in _SPECTRUM_REFERENCES: + if effective_bit_depth is not None or integration_time: + raise TypeError( + "Spectrum metadata only applies to WhiteSpectrum and TargetSpectrum references." + ) + if not isinstance(data, Measurement): + raise TypeError(f"Reference type {refType} takes a Measurement.") + if cuvis_il.status_ok != cuvis_il.cuvis_proc_cont_set_reference( + self._handle, data._handle, internal.__CuvisReferenceType__[refType] + ): + raise SDKException() + return + + if isinstance(data, Measurement): + raise TypeError( + f"Reference type {refType} takes spectrum data" + " (an ImageData with wavelengths, or a (wavelengths, values) pair)." + ) + wavelengths, values = _spectrum_arrays(data) + + if refType is ReferenceType.TargetSpectrum: + if effective_bit_depth is not None or integration_time: + raise TypeError( + "Counts metadata does not apply to the target spectrum." + ) + values = np.ascontiguousarray(values, dtype=np.float32) + if ( + cuvis_il.status_ok + != cuvis_il.cuvis_proc_cont_set_reference_target_spectrum_swig( + self._handle, wavelengths, values + ) + ): + raise SDKException() + return + + if effective_bit_depth is None: + raise ValueError( + "A white counts spectrum needs effective_bit_depth (1 to 16)." + ) + if values.min() < 0 or values.max() > 0xFFFF: + raise ValueError("White spectrum counts must fit uint16 (0 to 65535).") + values = np.ascontiguousarray(values, dtype=np.uint16) + if ( + cuvis_il.status_ok + != cuvis_il.cuvis_proc_cont_set_reference_white_spectrum_swig( + self._handle, + wavelengths, + values, + int(effective_bit_depth), + float(integration_time), + ) ): raise SDKException() pass @@ -75,6 +171,8 @@ def clear_reference(self, refType: ReferenceType) -> None: pass def get_reference(self, refType: ReferenceType) -> Measurement: + """The reference measurement, or None. Spectrum references are not measurements; + for those use get_reference_spectrum.""" has_ref = self.has_reference(refType) if not has_ref: return None @@ -85,6 +183,45 @@ def get_reference(self, refType: ReferenceType) -> Measurement: raise SDKException() return Measurement(cuvis_il.p_int_value(_ptr)) + def get_reference_spectrum(self, refType: ReferenceType) -> ImageData: + """The reference spectrum as an ImageData (shape 1 x 1 x channels, wavelengths in + nanometres), or None when the slot is empty. + + Only ReferenceType.WhiteSpectrum and ReferenceType.TargetSpectrum are spectra; + other types live in get_reference. For the white spectrum the returned ImageData + additionally carries the effective_bit_depth and integration_time it was set + with, as plain attributes (they do not survive slicing or arithmetic). + """ + if refType not in _SPECTRUM_REFERENCES: + raise ValueError( + f"Reference type {refType} is not a spectrum; use get_reference." + ) + if not self.has_reference(refType): + return None + if refType is ReferenceType.TargetSpectrum: + status, wavelengths, values = ( + cuvis_il.cuvis_proc_cont_get_reference_target_spectrum_swig( + self._handle + ) + ) + metadata = {} + else: + status, wavelengths, values, bit_depth, integration_time = ( + cuvis_il.cuvis_proc_cont_get_reference_white_spectrum_swig(self._handle) + ) + metadata = { + "effective_bit_depth": bit_depth, + "integration_time": integration_time, + } + if cuvis_il.status_ok != status: + raise SDKException() + spectrum = ImageData.from_array( + values, wavelength=[float(wl) for wl in wavelengths] + ) + for key, value in metadata.items(): + setattr(spectrum, key, value) + return spectrum + def has_reference(self, refType: ReferenceType) -> bool: _ptr = cuvis_il.new_p_int() if cuvis_il.status_ok != cuvis_il.cuvis_proc_cont_has_reference( diff --git a/cuvis/cuvis_types.py b/cuvis/cuvis_types.py index 2c1a1d7..2e9675e 100644 --- a/cuvis/cuvis_types.py +++ b/cuvis/cuvis_types.py @@ -85,6 +85,8 @@ class ReferenceType(Enum): WhiteDark = 3 SpRad = 4 Distance = 5 + WhiteSpectrum = 6 + TargetSpectrum = 7 __CuvisReferenceType__ = { @@ -93,6 +95,8 @@ class ReferenceType(Enum): ReferenceType.WhiteDark: cuvis_il.Reference_WhiteDark, ReferenceType.SpRad: cuvis_il.Reference_SpRad, ReferenceType.Distance: cuvis_il.Reference_Distance, + ReferenceType.WhiteSpectrum: cuvis_il.Reference_WhiteSpectrum, + ReferenceType.TargetSpectrum: cuvis_il.Reference_TargetSpectrum, } __ReferenceType__ = __inverseTranslationDict(__CuvisReferenceType__) diff --git a/tests/test_processing_context.py b/tests/test_processing_context.py index eb1b886..5f04c8c 100644 --- a/tests/test_processing_context.py +++ b/tests/test_processing_context.py @@ -5,7 +5,11 @@ processing modes, reference handling, and cube generation. """ +import numpy as np +import pytest + import cuvis +from cuvis.cuvis_aux import SDKException def test_processing_context_creation_from_session(processing_context_from_session): @@ -122,3 +126,254 @@ def test_cube_wavelength_access(processing_context_from_session, test_measuremen assert hasattr(cube, "wavelength") wavelength = cube.wavelength assert wavelength is not None + + +# --- reference spectra ----------------------------------------------------------------- +# The counts metadata (effective_bit_depth, integration_time) cannot be asserted after a +# round trip: the C API stores it but exposes no getter for it. + + +def _flat_target(n=10): + wavelengths = np.linspace(450.0, 900.0, n, dtype=np.float32) + values = np.full(n, 1.0, dtype=np.float32) + return wavelengths, values + + +def test_target_spectrum_round_trip(processing_context_from_session): + """A target spectrum set from numpy arrays reads back through get_reference_spectrum.""" + pc = processing_context_from_session + wavelengths, values = _flat_target() + + pc.set_reference((wavelengths, values), cuvis.ReferenceType.TargetSpectrum) + assert pc.has_reference(cuvis.ReferenceType.TargetSpectrum) + + spectrum = pc.get_reference_spectrum(cuvis.ReferenceType.TargetSpectrum) + assert isinstance(spectrum, cuvis.ImageData) + np.testing.assert_allclose( + np.asarray(spectrum.array).reshape(-1), values, rtol=1e-6 + ) + np.testing.assert_allclose(spectrum.wavelength, wavelengths, rtol=1e-6) + + pc.clear_reference(cuvis.ReferenceType.TargetSpectrum) + assert not pc.has_reference(cuvis.ReferenceType.TargetSpectrum) + assert pc.get_reference_spectrum(cuvis.ReferenceType.TargetSpectrum) is None + + +def test_white_counts_spectrum_round_trip(processing_context_from_session): + """A white counts spectrum with its bit depth reads back; values stay uint16.""" + pc = processing_context_from_session + wavelengths = np.linspace(450.0, 900.0, 8, dtype=np.float32) + counts = np.linspace(100, 4000, 8).astype(np.uint16) + + pc.set_reference( + (wavelengths, counts), + cuvis.ReferenceType.WhiteSpectrum, + effective_bit_depth=12, + integration_time=10.0, + ) + assert pc.has_reference(cuvis.ReferenceType.WhiteSpectrum) + + spectrum = pc.get_reference_spectrum(cuvis.ReferenceType.WhiteSpectrum) + assert np.asarray(spectrum.array).reshape(-1).tolist() == counts.tolist() + np.testing.assert_allclose(spectrum.wavelength, wavelengths, rtol=1e-6) + assert spectrum.effective_bit_depth == 12 + assert spectrum.integration_time == 10.0 + + pc.clear_reference(cuvis.ReferenceType.WhiteSpectrum) + assert not pc.has_reference(cuvis.ReferenceType.WhiteSpectrum) + + +def test_target_spectrum_from_image_data(processing_context_from_session): + """An ImageData built with from_array sets a target spectrum like the numpy pair.""" + pc = processing_context_from_session + wavelengths, values = _flat_target() + spectrum_in = cuvis.ImageData.from_array(values, wavelength=list(wavelengths)) + + pc.set_reference(spectrum_in, cuvis.ReferenceType.TargetSpectrum) + + spectrum_out = pc.get_reference_spectrum(cuvis.ReferenceType.TargetSpectrum) + np.testing.assert_allclose( + np.asarray(spectrum_out.array).reshape(-1), values, rtol=1e-6 + ) + pc.clear_reference(cuvis.ReferenceType.TargetSpectrum) + + +def test_white_counts_spectrum_requires_bit_depth(processing_context_from_session): + pc = processing_context_from_session + wavelengths, values = _flat_target() + with pytest.raises(ValueError, match="effective_bit_depth"): + pc.set_reference( + (wavelengths, values.astype(np.uint16)), cuvis.ReferenceType.WhiteSpectrum + ) + + +def test_spectrum_length_mismatch_rejected(processing_context_from_session): + pc = processing_context_from_session + with pytest.raises(ValueError, match="equally many"): + pc.set_reference( + (np.array([450.0, 500.0]), np.array([1.0])), + cuvis.ReferenceType.TargetSpectrum, + ) + + +def test_spectrum_and_measurement_slots_do_not_mix( + processing_context_from_session, test_measurement +): + """A Measurement cannot land in a spectrum slot and vice versa, and get_reference + does not serve spectrum slots.""" + pc = processing_context_from_session + wavelengths, values = _flat_target() + + with pytest.raises(TypeError): + pc.set_reference(test_measurement, cuvis.ReferenceType.TargetSpectrum) + with pytest.raises(TypeError): + pc.set_reference((wavelengths, values), cuvis.ReferenceType.Dark) + with pytest.raises(TypeError): + pc.set_reference( + test_measurement, cuvis.ReferenceType.Dark, effective_bit_depth=12 + ) + with pytest.raises(ValueError): + pc.get_reference_spectrum(cuvis.ReferenceType.Dark) + + pc.set_reference((wavelengths, values), cuvis.ReferenceType.TargetSpectrum) + with pytest.raises(SDKException): + pc.get_reference(cuvis.ReferenceType.TargetSpectrum) + pc.clear_reference(cuvis.ReferenceType.TargetSpectrum) + + +@pytest.mark.slow +def test_flat_target_spectrum_keeps_reflectance_identical( + processing_context_from_session, test_measurement +): + """A flat 1.0 (full reflectivity) target spectrum must not change the reflectance cube.""" + pc = processing_context_from_session + pc.processing_mode = cuvis.ProcessingMode.Reflectance + + pc.apply(test_measurement) + plain = np.array(test_measurement.data["cube"].array, copy=True) + + wavelengths, values = _flat_target(64) + pc.set_reference((wavelengths, values), cuvis.ReferenceType.TargetSpectrum) + pc.apply(test_measurement) + with_target = np.array(test_measurement.data["cube"].array, copy=True) + pc.clear_reference(cuvis.ReferenceType.TargetSpectrum) + + # one count of tolerance: the reflectance kernel is parallel and not + # bit-deterministic between runs, and a flat full-reflectivity target must only + # be a no-op within uint16 rounding + assert np.abs(plain.astype(np.int32) - with_target.astype(np.int32)).max() <= 1 + + +def test_target_spectrum_scales_reflectance( + processing_context_from_session, test_measurement +): + """A flat 0.5 reflectivity target spectrum halves every reflectance value.""" + pc = processing_context_from_session + pc.processing_mode = cuvis.ProcessingMode.Reflectance + pc.apply(test_measurement) + base = np.array(test_measurement.data["cube"].array, copy=True) + grid = np.asarray(test_measurement.data["cube"].wavelength, dtype=np.float32) + + pc.set_reference( + (grid, np.full(grid.size, 0.5, dtype=np.float32)), + cuvis.ReferenceType.TargetSpectrum, + ) + pc.apply(test_measurement) + halved = np.array(test_measurement.data["cube"].array, copy=True) + pc.clear_reference(cuvis.ReferenceType.TargetSpectrum) + + mask = base > 200 # above the noise floor, so uint16 rounding stays below 1 percent + ratio = halved[mask].astype(np.float64) / base[mask] + assert abs(ratio.mean() - 0.5) < 0.005 + assert ratio.min() > 0.49 and ratio.max() < 0.51 + + +def _padded_white(test_measurement, level): + """A flat counts spectrum spanning past the cube grid, whose rounded ends may sit + inside the calibration's float grid.""" + grid = np.asarray(test_measurement.data["cube"].wavelength, dtype=np.float32) + wls = np.concatenate(([grid[0] - 10.0], grid, [grid[-1] + 10.0])).astype(np.float32) + return wls, np.full(wls.size, level, dtype=np.uint16) + + +def test_white_spectrum_replaces_white_reference( + processing_context_from_session, test_measurement +): + """The two white sources are mutually exclusive: setting one clears the other, and + reflectance uses whichever is set; four times the counts quarter the cube.""" + pc = processing_context_from_session + pc.processing_mode = cuvis.ProcessingMode.Reflectance + pc.apply(test_measurement) + assert pc.has_reference(cuvis.ReferenceType.White) + white_mesu = pc.get_reference(cuvis.ReferenceType.White) + + wls, counts = _padded_white(test_measurement, 1000) + pc.set_reference( + (wls, counts), cuvis.ReferenceType.WhiteSpectrum, effective_bit_depth=12 + ) + assert not pc.has_reference(cuvis.ReferenceType.White) + assert pc.has_reference(cuvis.ReferenceType.WhiteSpectrum) + + pc.apply(test_measurement) + from_1000 = np.array(test_measurement.data["cube"].array, copy=True) + + wls, counts = _padded_white(test_measurement, 4000) + pc.set_reference( + (wls, counts), cuvis.ReferenceType.WhiteSpectrum, effective_bit_depth=12 + ) + pc.apply(test_measurement) + from_4000 = np.array(test_measurement.data["cube"].array, copy=True) + + mask = from_1000 > 200 + ratio = from_4000[mask].astype(np.float64) / from_1000[mask] + assert abs(ratio.mean() - 0.25) < 0.005 + + # and the other direction: restoring the white measurement clears the spectrum + pc.set_reference(white_mesu, cuvis.ReferenceType.White) + assert pc.has_reference(cuvis.ReferenceType.White) + assert not pc.has_reference(cuvis.ReferenceType.WhiteSpectrum) + + +def test_spectra_survive_legacy_cu3_round_trip( + processing_context_from_session, test_measurement, temp_output_dir +): + """A measurement processed with both spectra keeps them through a legacy .cu3 save + behind the SpectrumFileRefData indirection: one .cu3sp sidecar per slot beside the + .cu3, never embedded, and the stored cube is bit-identical after reload.""" + pc = processing_context_from_session + pc.processing_mode = cuvis.ProcessingMode.Reflectance + pc.apply(test_measurement) + grid = np.asarray(test_measurement.data["cube"].wavelength, dtype=np.float32) + target = np.full(grid.size, 50.0, dtype=np.float32) + pc.set_reference((grid, target), cuvis.ReferenceType.TargetSpectrum) + wls, counts = _padded_white(test_measurement, 1000) + pc.set_reference( + (wls, counts), cuvis.ReferenceType.WhiteSpectrum, effective_bit_depth=12 + ) + pc.apply(test_measurement) + expected = np.array(test_measurement.data["cube"].array, copy=True) + + # the CubeExporter is the one component that writes references in their link + # forms; allow_session_file=False makes it produce a loose legacy .cu3 + exporter = cuvis.CubeExporter( + cuvis.SaveArgs( + export_dir=str(temp_output_dir), + allow_overwrite=True, + allow_session_file=False, + allow_info_file=False, + full_export=True, + ) + ) + exporter.apply(test_measurement) + exporter.flush() + del exporter + cu3 = list(temp_output_dir.glob("*.cu3")) + assert len(cu3) == 1 + + # the spectra land as sidecar files beside the other reference files, shared by + # every measurement saved into the directory, never embedded into the measurement + assert (temp_output_dir / "Calibration" / "target_spectrum_ref.cu3sp").is_file() + assert (temp_output_dir / "Calibration" / "white_spectrum_ref.cu3sp").is_file() + + reloaded = cuvis.Measurement(str(cu3[0])) + assert np.array_equal(np.asarray(reloaded.data["cube"].array), expected)