From a99c52ac14b8cfa824df920df41954ef821744e9 Mon Sep 17 00:00:00 2001 From: Francois Drielsma Date: Tue, 3 Nov 2026 13:22:09 -0800 Subject: [PATCH 1/5] Add native pysupera HDF5 input support --- config/io/supera.yaml | 23 + docs/source/api/io.rst | 150 +++++ pyproject.toml | 3 + src/spine/io/dataset/__init__.py | 2 + src/spine/io/dataset/hdf5.py | 3 +- src/spine/io/dataset/supera.py | 57 ++ src/spine/io/parse/supera/__init__.py | 3 + src/spine/io/parse/supera/truth.py | 587 ++++++++++++++++++ src/spine/io/read/__init__.py | 3 +- src/spine/io/read/base.py | 9 +- src/spine/io/read/cache.py | 4 +- src/spine/io/read/larcv.py | 4 +- src/spine/io/read/supera.py | 267 ++++++++ test/test_io/test_supera/coverage.ini | 13 + test/test_io/test_supera/test_supera.py | 402 ++++++++++++ test/test_io/test_supera/test_supera_model.py | 117 ++++ 16 files changed, 1636 insertions(+), 11 deletions(-) create mode 100644 config/io/supera.yaml create mode 100644 src/spine/io/dataset/supera.py create mode 100644 src/spine/io/parse/supera/__init__.py create mode 100644 src/spine/io/parse/supera/truth.py create mode 100644 src/spine/io/read/supera.py create mode 100644 test/test_io/test_supera/coverage.ini create mode 100644 test/test_io/test_supera/test_supera.py create mode 100644 test/test_io/test_supera/test_supera_model.py diff --git a/config/io/supera.yaml b/config/io/supera.yaml new file mode 100644 index 00000000..13c27459 --- /dev/null +++ b/config/io/supera.yaml @@ -0,0 +1,23 @@ +# Native pysupera 4.1 truth-deposit input for segmentation, PPN and fragmentation. +# Merge this io block into a training configuration; no conversion is required. +io: + loader: + batch_size: 1 + num_workers: 0 + shuffle: true + dataset: + provider: supera + file_keys: supera_pixel_j000000-000004.h5 + schema: + data: + provider: tensor + seg_label: + provider: segmentation + clust_label: + provider: cluster_label + ppn_label: + provider: particle_points + particles: + provider: particle + collate_fn: + provider: all diff --git a/docs/source/api/io.rst b/docs/source/api/io.rst index 7deb26a8..52302f14 100644 --- a/docs/source/api/io.rst +++ b/docs/source/api/io.rst @@ -44,9 +44,158 @@ File Readers :toctree: generated read.HDF5Reader + read.SuperaReader read.LArCVReader read.CacheReader +Native pysupera input +--------------------- + +Use ``provider: supera`` for pysupera **4.1.0** files. This is a separate +format from SPINE HDF5 V1/V2; ``provider: hdf5`` does not read these files. +Install ``spine[supera]`` for the HDF5 compression filters used by upstream +productions. No pysupera or LArCV runtime is required. + +.. code-block:: yaml + + io: + loader: + batch_size: 1 + num_workers: 0 + dataset: + provider: supera + file_keys: supera_pixel_j000000-000004.h5 + schema: + data: {provider: tensor} + seg_label: {provider: segmentation} + clust_label: {provider: cluster_label} + ppn_label: {provider: particle_points} + particles: {provider: particle} + collate_fn: {provider: all} + +``SuperaDataset`` reuses the HDF5 dataset's parsing, collation and augmentation +interface with a format-specific parser registry. ``SuperaReader`` supports +multiple files, ordinary entry selection, source-file provenance and projected +reads. Its default raw products are ``points``, ``particles``, ``inter``, +``events`` and ``metadata``. Particle and interaction point ranges are rebased +to the event's point array. Interaction particle ranges are likewise rebased. +Event identities remain available in the raw ``events`` product, including +production ``event_id`` and ``job_id`` when present; these are not invented +run/subrun numbers. + +Physical semantics +~~~~~~~~~~~~~~~~~~ + +The implementation was checked against `pysupera commit dab92d8 +`_, +in particular ``_run.py``, ``layout.py``, ``io_v3.py``, ``provenance.py`` and +``torchdata.py``: + +* With JAXTPC input, ``points/flat`` contains **truth energy deposits**, not + measured hits. Coordinates are mm, ``t`` is microseconds, ``dE`` is MeV, + ``dX`` is cm, angles are radians and ``p`` is MeV/c. ``hit_energy=charge`` + describes the hit-based partitioning input; it does not change ``dE`` into + charge. Measured hit coordinates and charge require the separate JAXTPC file. +* ``groups`` is indexed by JAXTPC deposit group, concatenated across volumes. + Its rows do not align with ``points/flat``. In the supplied 1,000-event + production there are 54,863,490 groups and 31,871,012 point rows. + ``volume_offsets`` identifies each volume's start in the event's group array. +* Fragment representatives satisfy ``frag_id == id``. Their half-open + ``frag_pc_start:frag_pc_end`` ranges partition the point rows. Instance ranges + also partition the points, with non-LE fragments preceding the LE portion + starting at ``inst_pc_le_start``. ``frag_inst_id`` joins each fragment to + its instance. The reader never assumes group/point alignment. +* The writer retains sparse particle IDs. Despite the upstream reader module's + row-index wording, these must be resolved through an ID-to-row map. SPINE + particle indexes, ancestor links and PPN particle references are event-local + table indexes; the cluster sidecar's ``particle`` field retains source IDs. +* Semantic codes map explicitly: pysupera shower/track/delta/Michel/LE + ``1/2/3/4/5`` become SPINE ``0/1/3/2/4``. Non-LE points use the instance's + semantic class; LE fragments retain LE even inside a track or shower. + Unknown point semantics are rejected by the five-class training parsers. +* Particle ``start_*``/``end_*`` describe Geant4 creation/destruction. + Instance ``first_*``/``last_*`` describe the first/last collected steps, + subject to ``kinematics_steps`` (``visible`` in the sample). Kinematic + positions are mm and times ns. Showers have no last endpoint. + +Aligned training products +~~~~~~~~~~~~~~~~~~~~~~~~~ + +All parsers share one event preparation. Coordinates are rasterized with the +file's ``voxel_size`` and an origin of zero, then shifted by an integer number +of cells to make event coordinates nonnegative. ``Meta`` records the inverse +transform in cm. Each parser accepts ``voxel_size_mm`` (scalar or three-vector) +and ``origin_mm``; all parsers in a dataset must use the same grid settings. +An explicit size is required when the file does not supply one. These settings +also permit rasterizing deposit-mode files. Upstream voxelization can use a +per-particle origin, so stored centers are rasterized rather than assumed to +already be integer coordinates on a common grid. + +Overlapping contributions sum their energy. Label ownership follows upstream +priority (track, shower, delta, Michel, LE), then earliest deposit time, then +source row order. The same winning contribution selects semantic, fragment +and instance labels. LE deposits are retained. ``ClusterLabelData`` stores +fragment IDs and fragment-row associations, with instance IDs in ``group``. +Unprovided neutrino identity is left at ``-1``. No energy-to-charge conversion +is performed. + +PPN targets use finite instance starts and track ends only, exclude LE, +unknown classes and nuclear fragments, and omit points outside the event +image. Targets refer to the complete event particle table. The particle parser +returns that table as SPINE ``Particle`` objects in voxel units, preserving +missing kinematics as NaN. Particle energy deposits sum the source instance's +contributions, independently of overlap winner selection. + +``groups`` and ``hit_labels/volume0/plane0`` (or another explicit plane path) +can be requested through ``SuperaReader(keys=...)``. They are never read by +the training parsers or by default. Even explicitly requested hit labels are +sliced by event offsets; no whole-plane label array is materialized. + +Validation +~~~~~~~~~~ + +The synthetic tests cover sparse IDs, fragment/instance distinctions, LE, +semantic remapping, overlapping energy, empty events, multi-file selection, +absolute-range rebasing, pickle round trips and projected hit reads. Error-path +tests also cover invalid grids, corrupt relationships, nonfinite values, +missing compression support and endpoint filtering. The dedicated coverage +configuration requires 100% line and branch coverage of the Supera modules, +without excluding defensive or optional-dependency paths: + +.. code-block:: bash + + PYTHONPATH=src pytest -q test/test_io/test_supera \ + --cov=src/spine --cov-config=test/test_io/test_supera/coverage.ini \ + --cov-report=term-missing + +Run the real-file forward/backward test in the model runtime with: + +.. code-block:: bash + + docker run --rm --platform linux/amd64 \ + -e PYTHONPATH=/workspace/src -e OMP_NUM_THREADS=1 \ + -e SPINE_SUPERA_SAMPLE=/workspace/supera_pixel_j000000-000004.h5 \ + -v "$PWD:/workspace" -w /workspace \ + ghcr.io/deeplearnphysics/spine:latest bash -lc \ + 'pip install hdf5plugin && pytest -q -s test/test_io/test_supera/test_supera_model.py' + +The first sample event yields 25,537 unique voxels and 120 PPN targets. The +integration test exercises segmentation and PPN through the standard +UResNet+PPN model and fragmentation through Graph-SPICE, using the full event. +It requires finite, nonzero backbone gradients for each objective and verifies +that the combined backward pass reaches the graph kernel. Both models are +available on ``main``; this input implementation has no dependency on the +separate UResNetMulti feature branch. + +.. autosummary:: + :toctree: generated + + parse.supera.SuperaTensorParser + parse.supera.SuperaSegmentationParser + parse.supera.SuperaClusterLabelParser + parse.supera.SuperaParticleParser + parse.supera.SuperaParticlePointsParser + Entry Filtering --------------- @@ -234,6 +383,7 @@ with raw input through the mixed dataset. dataset.LArCVDataset dataset.CacheDataset dataset.HDF5Dataset + dataset.SuperaDataset dataset.MixedDataset dataset.JointDataset diff --git a/pyproject.toml b/pyproject.toml index 75477265..484752c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,9 @@ Documentation = "https://spine.readthedocs.io/" "Bug Tracker" = "https://github.com/DeepLearnPhysics/spine/issues" [project.optional-dependencies] +# Compression filters used by native pysupera files +supera = ["hdf5plugin>=4"] + # Visualization dependencies viz = [ "matplotlib>=3.3.0", diff --git a/src/spine/io/dataset/__init__.py b/src/spine/io/dataset/__init__.py index b0ba5b72..b4cf2668 100644 --- a/src/spine/io/dataset/__init__.py +++ b/src/spine/io/dataset/__init__.py @@ -15,6 +15,7 @@ from .joint import JointDataset from .larcv import LArCVDataset from .mixed import MixedDataset +from .supera import SuperaDataset __all__ = [ "CacheDataset", @@ -22,4 +23,5 @@ "JointDataset", "LArCVDataset", "MixedDataset", + "SuperaDataset", ] diff --git a/src/spine/io/dataset/hdf5.py b/src/spine/io/dataset/hdf5.py index 4ef92280..12e8900b 100644 --- a/src/spine/io/dataset/hdf5.py +++ b/src/spine/io/dataset/hdf5.py @@ -30,6 +30,7 @@ class HDF5Dataset(BaseDataset): name: ClassVar[str] = "hdf5" supports_stages: ClassVar[bool] = False + parser_dict: ClassVar[dict[str, Any]] = PARSER_DICT parsers: dict[str, Any] reader: ReaderBase @@ -102,7 +103,7 @@ def __init__( parser_cfg = dict(parser_cfg) parser_stage = parser_cfg.pop("stage", stage) parser = instantiate( - PARSER_DICT, parser_cfg, alt_name="parser", dtype=dtype + self.parser_dict, parser_cfg, alt_name="parser", dtype=dtype ) self.parsers[data_product] = parser for key in parser.tree_keys: diff --git a/src/spine/io/dataset/supera.py b/src/spine/io/dataset/supera.py new file mode 100644 index 00000000..0e33ef1f --- /dev/null +++ b/src/spine/io/dataset/supera.py @@ -0,0 +1,57 @@ +"""Native pysupera dataset using the common HDF5 parsing machinery.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, ClassVar + +from spine.config.factory import module_dict + +from ..parse import supera +from ..read.supera import SuperaReader +from .hdf5 import HDF5Dataset + +__all__ = ["SuperaDataset"] + + +class SuperaDataset(HDF5Dataset): + """Parse pysupera truth deposits into aligned SPINE training products.""" + + name: ClassVar[str] = "supera" + parser_dict = module_dict(supera) + + def _build_reader( + self, stage: str | None, stage_map: Mapping[str, str], kwargs: Mapping[str, Any] + ) -> SuperaReader: + """Select the native reader and enforce a common voxelization grid. + + Parameters + ---------- + stage : str, optional + Reserved cache-stage selection; always None for this dataset. + stage_map : mapping + Reserved cache product routing; empty for this dataset. + kwargs : mapping + Physical-reader and entry-selection options. + + Returns + ------- + SuperaReader + Native reader projected onto the products required by the schema. + + Raises + ------ + ValueError + If parsers request inconsistent grid settings. + """ + del stage, stage_map + + # Every supervision product must use the same voxel coordinates. + grids = {p.grid for p in self.parsers.values() if hasattr(p, "grid")} + if len(grids) > 1: + raise ValueError("Supera parsers must use the same voxel size and origin.") + + # Reuse the parent dataset's schema-driven physical projection. + return SuperaReader( + keys=None if self.keys is None else tuple(self.keys), **kwargs + ) diff --git a/src/spine/io/parse/supera/__init__.py b/src/spine/io/parse/supera/__init__.py new file mode 100644 index 00000000..5c2c4fbb --- /dev/null +++ b/src/spine/io/parse/supera/__init__.py @@ -0,0 +1,3 @@ +"""Parsers for native pysupera truth-deposit products.""" + +from .truth import * diff --git a/src/spine/io/parse/supera/truth.py b/src/spine/io/parse/supera/truth.py new file mode 100644 index 00000000..b7aa9b3c --- /dev/null +++ b/src/spine/io/parse/supera/truth.py @@ -0,0 +1,587 @@ +"""Aligned voxel, fragment, particle and point-proposal supervision.""" + +from __future__ import annotations + +import json +from typing import Any + +import numpy as np + +from spine.constants import PDG_TO_PID +from spine.data import ClusterLabelData, Meta, ObjectListData, Particle, TensorData + +from ..base import ParserBase + +__all__ = [ + "SuperaClusterLabelParser", + "SuperaParticleParser", + "SuperaParticlePointsParser", + "SuperaSegmentationParser", + "SuperaTensorParser", +] + + +def _shape(code: int | np.ndarray) -> np.ndarray: + """Translate one-based pysupera classes to SPINE semantic labels. + + Parameters + ---------- + code : int or np.ndarray + Source semantic code or array of codes. + + Returns + ------- + np.ndarray + SPINE labels of the same shape; unknown codes map to ``-1``. + """ + code = np.asarray(code) + return np.array([-1, 0, 1, 3, 2, 4, -1])[np.clip(code, 0, 6)] + + +class SuperaTensorParser(ParserBase): + """Voxelize truth deposits once, sharing winner selection across products. + + Coordinates and optional grid arguments are in mm; output metadata uses + cm. The default size comes from the file. The origin defaults to (0,0,0); + an event-local integer translation makes all voxel coordinates nonnegative. + Overlaps sum dE and choose labels by track, shower, delta, Michel, LE, + then earliest time and source row order. Unknown point classes are rejected. + """ + + name = "tensor" + + def __init__( + self, + dtype: str, + voxel_size_mm: float | tuple[float, ...] | None = None, + origin_mm: tuple[float, ...] = (0.0, 0.0, 0.0), + ) -> None: + """Configure a common grid for all parsers in the dataset. + + Parameters + ---------- + dtype : str + Floating-point type used by parser products. + voxel_size_mm : float or sequence[float], optional + Rasterization size; defaults to the file's voxel_size attribute. + origin_mm : sequence[float], default (0, 0, 0) + Common grid origin in detector coordinates (mm). + """ + super().__init__( + dtype, + points_event="points", + particle_event="particles", + metadata_event="metadata", + ) + + # Normalize grid settings so all parsers can share the event cache. + self.size = ( + None + if voxel_size_mm is None + else tuple(np.broadcast_to(np.asarray(voxel_size_mm, dtype=float), (3,))) + ) + self.origin = tuple(np.asarray(origin_mm, dtype=float)) + if len(self.origin) != 3 or not np.isfinite(self.origin).all(): + raise ValueError("origin_mm must contain three finite values.") + if self.size is not None and ( + not np.isfinite(self.size).all() or min(self.size) <= 0 + ): + raise ValueError("voxel_size_mm must be finite and positive.") + self.grid = (self.size, self.origin) + + def prepare(self, trees: dict[str, Any]) -> dict[str, Any]: + """Build and cache the common voxel representation for all products. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products. A private cache is added to this dictionary. + + Returns + ------- + dict[str, Any] + Unique voxel coordinates, summed energy, winning labels, spatial + metadata and the resolved particle table. + + Raises + ------ + ValueError + If the grid, particle associations or point values are invalid. + """ + cache = trees.setdefault("_supera_prepared", {}) + if self.grid in cache: + return cache[self.grid] + + # Prefer an explicit grid size; otherwise decode the file attribute. + points, table, attrs = trees["points"], trees["particles"], trees["metadata"] + size = self.size + if size is None: + raw = attrs.get("voxel_size") + if raw is None: + raise ValueError( + "Missing voxel_size: configure voxel_size_mm explicitly." + ) + size = json.loads(raw) if isinstance(raw, (str, bytes)) else raw + size = np.broadcast_to(np.asarray(size, dtype=float), (3,)) + if not np.isfinite(size).all() or np.any(size <= 0): + raise ValueError("Invalid voxel size.") + # Resolve sparse source IDs before expanding fragment and instance ranges. + source_ids = table["id"].copy() + table = self._resolve_particles(table) + fragment, instance, semantic = self._point_labels(table, len(points)) + + # Select labels using the same overlap winners as the input tensor. + event = self._voxelize(points, size, attrs, semantic) + winner = event.pop("winner") + event.update( + shape=semantic[winner], + fragment=fragment[winner], + instance=instance[winner], + columns=table, + source_ids=source_ids, + ) + cache[self.grid] = event + return event + + def _voxelize( + self, + points: np.ndarray, + size: np.ndarray, + attrs: dict[str, Any], + semantic: np.ndarray, + ) -> dict[str, Any]: + """Rasterize deposits and retain their overlap winners and transform. + + Parameters + ---------- + points : np.ndarray + Event point rows in detector coordinates. + size : np.ndarray + Three voxel dimensions in mm. + attrs : dict[str, Any] + File attributes, including point column names. + semantic : np.ndarray + SPINE semantic label of each point. + + Returns + ------- + dict[str, Any] + Unique coordinates, summed energy, winning source row and Meta. + """ + num_points = len(points) + # Rasterize on a common grid and translate to nonnegative coordinates. + if not np.isfinite(points[:, :3]).all(): + raise ValueError("Nonfinite point coordinates.") + cells = np.floor((points[:, :3] - self.origin) / size).astype(np.int64) + lower_cell = cells.min(axis=0) if num_points else np.zeros(3, dtype=np.int64) + coords, inverse = np.unique(cells - lower_cell, axis=0, return_inverse=True) + inverse = inverse.reshape(-1) + + energy, winner = self._merge_voxels(points, attrs["columns"], semantic, inverse) + + # Metadata preserves the detector transform in SPINE centimetres. + lower = (np.asarray(self.origin) + lower_cell * size) / 10.0 + count = coords.max(axis=0) + 1 if num_points else np.ones(3, dtype=np.int64) + meta = Meta( + lower=lower, + upper=lower + count * size / 10.0, + size=size / 10.0, + count=count, + ) + return {"coords": coords, "energy": energy, "winner": winner, "meta": meta} + + @staticmethod + def _merge_voxels( + points: np.ndarray, + columns: tuple[str, ...], + semantic: np.ndarray, + inverse: np.ndarray, + ) -> tuple[np.ndarray, np.ndarray]: + """Sum energy and resolve overlap labels by class, time and row order. + + Parameters + ---------- + points : np.ndarray + Event point rows before duplicate removal. + columns : tuple[str, ...] + Physical column names in the point array. + semantic : np.ndarray + SPINE semantic label for each contribution. + inverse : np.ndarray + Unique voxel index of each contribution. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + Summed voxel energies and winning source point indexes. + """ + num_points = len(points) + values = points[:, columns.index("dE")] + times = points[:, columns.index("t" if "t" in columns else "time")] + if not np.isfinite(values).all() or not np.isfinite(times).all(): + raise ValueError("Nonfinite deposit energy or time.") + ranks = np.full(num_points, 5) + for rank, shape in enumerate((1, 0, 3, 2, 4)): + ranks[semantic == shape] = rank + order = np.lexsort((np.arange(num_points), times, ranks, inverse)) + winner = ( + order[np.r_[True, np.diff(inverse[order]) != 0]] if num_points else order + ) + energy = np.bincount(inverse, weights=values, minlength=0) + + return energy, winner + + @staticmethod + def _resolve_particles(table: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + """Resolve sparse particle relationships to event-local table indexes. + + Parameters + ---------- + table : dict[str, np.ndarray] + Raw particle columns with source IDs and rebased point ranges. + + Returns + ------- + dict[str, np.ndarray] + Shallow copy with relationship columns replaced by row indexes. + + Raises + ------ + ValueError + If particle IDs are invalid or a relationship cannot be resolved. + """ + source_ids = table["id"] + num_particles = len(source_ids) + if len(np.unique(source_ids)) != num_particles or np.any(source_ids < 0): + raise ValueError("Supera particle ids must be unique and nonnegative.") + # Contrary to io_v3's index-convention prose, layout.py retains sparse + # source IDs. Resolve relationships explicitly, never index by an ID. + row_of = {int(pid): row for row, pid in enumerate(source_ids)} + table = dict(table) + for key in ( + "id", + "parent_id", + "ancestor_id", + "frag_id", + "inst_id", + "frag_inst_id", + ): + try: + table[key] = np.array( + [row_of[int(v)] if v >= 0 else -1 for v in table[key]], + dtype=np.int64, + ) + except KeyError as err: + raise ValueError( + f"Dangling Supera particle relationship in {key}." + ) from err + return table + + @staticmethod + def _point_labels( + table: dict[str, np.ndarray], num_points: int + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Expand and validate fragment and instance ownership of every point. + + Parameters + ---------- + table : dict[str, np.ndarray] + Particle columns with event-local relationships and point ranges. + num_points : int + Number of point rows in this event. + + Returns + ------- + tuple[np.ndarray, np.ndarray, np.ndarray] + Fragment row, instance row and SPINE semantic label for each point. + + Raises + ------ + ValueError + If ranges overlap, leave gaps or disagree with stored relationships + or semantic classes. + """ + # Each representative owns one half-open range; no point is shared. + fragment = np.full(num_points, -1, dtype=np.int64) + instance = np.full(num_points, -1, dtype=np.int64) + semantic = np.full(num_points, -1, dtype=np.int64) + for level, dest in (("frag", fragment), ("inst", instance)): + for row in np.flatnonzero(table[level + "_id"] == table["id"]): + start, stop = int(table[level + "_pc_start"][row]), int( + table[level + "_pc_end"][row] + ) + if start == stop == -1: + continue + if not 0 <= start <= stop <= num_points or np.any( + dest[start:stop] >= 0 + ): + raise ValueError(f"Invalid or overlapping {level} point ranges.") + dest[start:stop] = row + if level == "inst": + split = int(table["inst_pc_le_start"][row]) + if not start <= split <= stop: + raise ValueError("Invalid instance LE range.") + semantic[start:split] = _shape(table["inst_sem_type"][row]) + semantic[split:stop] = 4 + + # Cross-check both partitions and the explicit fragment-to-instance join. + if np.any(fragment < 0) or np.any(instance < 0): + raise ValueError("Fragment and instance ranges must cover every point.") + if np.any(semantic < 0): + raise ValueError( + "Unknown Supera semantic class cannot supervise a five-class model." + ) + if num_points and not np.array_equal(table["frag_inst_id"][fragment], instance): + raise ValueError("Fragment ownership disagrees with instance ranges.") + if num_points and not np.array_equal( + table["frag_sem_type"][fragment] == 5, semantic == 4 + ): + raise ValueError("Fragment semantics disagree with instance LE ranges.") + return fragment, instance, semantic + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return summed truth energy in MeV on the shared sparse grid. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + TensorData + Sparse coordinates and summed deposit energy in MeV. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + return TensorData( + coords=event["coords"].astype(self.itype), + features=event["energy"].astype(self.ftype)[:, None], + meta=event["meta"], + ) + + +class SuperaSegmentationParser(SuperaTensorParser): + """Return semantic labels in exactly the input voxel order.""" + + name = "segmentation" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return five-class labels; unknown point semantics are rejected. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + TensorData + Semantic labels sharing the input tensor coordinates and row order. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + return TensorData( + coords=event["coords"].astype(self.itype), + features=event["shape"].astype(self.ftype)[:, None], + meta=event["meta"], + ) + + +class SuperaClusterLabelParser(SuperaTensorParser): + """Return fragment IDs and the winning fragment's particle table row.""" + + name = "cluster_label" + + def __call__(self, trees: dict[str, Any]) -> ClusterLabelData: + """Build compact fragment supervision with instance and ancestor links. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + ClusterLabelData + Compact fragment labels and a named particle-truth sidecar. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table = event["columns"] + num_particles = len(table["id"]) + host = table["frag_inst_id"] + valid = (host >= 0) & (host < num_particles) + shapes = _shape(table["sem_type"]) + shapes[valid] = _shape(table["inst_sem_type"][host[valid]]) + shapes[table["frag_sem_type"] == 5] = 4 + + # Keep source particle IDs while exposing event-local relationship fields. + particles = { + "particle": event["source_ids"].copy(), + "group": host.copy(), + "ancestor": table["ancestor_id"].copy(), + "interaction": table["interaction_id"].copy(), + "nu": np.full(num_particles, -1), + "pid": np.array( + [PDG_TO_PID.get(int(particle), -1) for particle in table["pdg"]] + ), + "group_primary": np.where(valid, table["id"] == host, -1), + "interaction_primary": (table["parent_id"] == table["id"]).astype(np.int64), + "vertex": ( + table["start_pos"][table["ancestor_id"], :3] / 10.0 + - event["meta"].lower + ) + / event["meta"].size, + "momentum": np.linalg.norm(table["start_mom"][:, :3], axis=1), + "energy_init": table["start_mom"][:, 3].copy(), + "shape": shapes, + } + + # The winning fragment row is also the compact particle association. + return ClusterLabelData( + coords=event["coords"].astype(self.itype), + features=np.column_stack( + (event["energy"], event["fragment"], event["fragment"]) + ).astype(self.ftype), + particles=particles, + meta=event["meta"], + remove_duplicates=False, + ) + + +class SuperaParticleParser(SuperaTensorParser): + """Translate the full stored particle table, preserving event-local IDs.""" + + name = "particle" + + def __call__(self, trees: dict[str, Any]) -> ObjectListData: + """Return SPINE particle truth in voxel units with MeV kinematics. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + ObjectListData + Complete stored particle table in voxel units with MeV kinematics. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table, meta = event["columns"], event["meta"] + particles = [] + + # Deposited truth energy includes source contributions, not winner totals. + event_counts = np.bincount(event["instance"], minlength=len(table["id"])) + energy_prefix = np.r_[ + 0.0, + np.cumsum( + trees["points"][:, trees["metadata"]["columns"].index("dE")], + dtype=np.float64, + ), + ] + + # Preserve the full table so PPN and cluster references remain valid. + for row in range(len(table["id"])): + is_inst = table["inst_id"][row] == row + particle = Particle( + id=row, + parent_id=int(table["parent_id"][row]), + group_id=int(table["frag_inst_id"][row]), + interaction_id=int(table["interaction_id"][row]), + track_id=int(table["geant4_trackid"][row]), + pdg_code=int(table["pdg"][row]), + parent_pdg_code=int(table["parent_pdg"][row]), + shape=int( + _shape(table["inst_sem_type" if is_inst else "sem_type"][row]) + ), + energy_init=float(table["start_mom"][row, 3]), + momentum=table["start_mom"][row, :3].copy(), + units="px", + ) + + # Positions use voxel units; kinematic times remain in nanoseconds. + for source, target in ( + ("start", "position"), + ("end", "end_position"), + ("first", "first_step"), + ("last", "last_step"), + ): + setattr( + particle, + target, + ( + (table[source + "_pos"][row, :3] / 10.0 - meta.lower) + / meta.size + ).astype(self.ftype), + ) + particle.t, particle.end_t = float(table["start_pos"][row, 3]), float( + table["end_pos"][row, 3] + ) + particle.first_step_t, particle.last_step_t = float( + table["first_pos"][row, 3] + ), float(table["last_pos"][row, 3]) + particle.num_voxels = int(event_counts[row]) + if is_inst: + start, stop = table["inst_pc_start"][row], table["inst_pc_end"][row] + particle.energy_deposit = ( + float(energy_prefix[stop] - energy_prefix[start]) + if start >= 0 + else 0.0 + ) + particles.append(particle) + return ObjectListData(particles, Particle()) + + +class SuperaParticlePointsParser(SuperaTensorParser): + """PPN instance starts and track ends from collected-step kinematics.""" + + name = "particle_points" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Omit unknown, LE, nonfinite and out-of-image endpoints. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + TensorData + Instance starts and track ends with shape, particle and endpoint labels. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table, meta = event["columns"], event["meta"] + rows = [] + + # Only instances represented in the voxel image can provide targets. + for row in np.unique(event["instance"]): + shape = int(_shape(table["inst_sem_type"][row])) + if shape not in (0, 1, 2, 3) or abs(table["pdg"][row]) > 1000000000: + continue + for endpoint, key in enumerate( + ("first_pos", "last_pos") if shape == 1 else ("first_pos",) + ): + point = (table[key][row, :3] / 10.0 - meta.lower) / meta.size + if ( + np.isfinite(point).all() + and np.all(point >= 0) + and np.all(point < meta.count) + ): + rows.append([*point, shape, row, endpoint]) + + # Preserve a stable six-column layout even when no endpoints survive. + data = np.asarray(rows, dtype=self.ftype).reshape(-1, 6) + return TensorData( + coords=data[:, :3], + features=data[:, 3:], + meta=meta, + coordinate_groups={"point": (0, 1, 2)}, + feature_fields={"shape": (0,), "particle": (1,), "endpoint": (2,)}, + index_cols=np.array([1]), + index_shifts=np.array([len(table["id"])]), + ) diff --git a/src/spine/io/read/__init__.py b/src/spine/io/read/__init__.py index 7abc6216..6cd56b71 100644 --- a/src/spine/io/read/__init__.py +++ b/src/spine/io/read/__init__.py @@ -3,5 +3,6 @@ from .cache import CacheReader from .hdf5 import HDF5Reader from .larcv import LArCVReader +from .supera import SuperaReader -__all__ = ["CacheReader", "HDF5Reader", "LArCVReader"] +__all__ = ["CacheReader", "HDF5Reader", "LArCVReader", "SuperaReader"] diff --git a/src/spine/io/read/base.py b/src/spine/io/read/base.py index 3e4c618c..cbfe8a98 100644 --- a/src/spine/io/read/base.py +++ b/src/spine/io/read/base.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from collections.abc import Sequence from numbers import Real -from typing import Any +from typing import Any, ClassVar import numpy as np @@ -45,11 +45,12 @@ class ReaderBase(ABC): Maps each available (run, subrun, event) triplet onto an entry_index index """ - name: str + name: ClassVar[str] file_paths: list[str] file_index: np.ndarray file_offsets: np.ndarray entry_index: np.ndarray + eligible_entry_index: np.ndarray num_entries: int run_info: list[tuple[int, int, int]] | None = None run_map: dict[tuple[int, int, int], int] | None = None @@ -199,9 +200,7 @@ def is_remote_path(path: str) -> bool: bool `True` if the path is a supported remote URI """ - return isinstance(path, str) and ( - path.startswith("root://") or path.startswith("xroot://") - ) + return isinstance(path, str) and (path.startswith(("root://", "xroot://"))) def process_file_paths( self, diff --git a/src/spine/io/read/cache.py b/src/spine/io/read/cache.py index 964bc4ac..4be84686 100644 --- a/src/spine/io/read/cache.py +++ b/src/spine/io/read/cache.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import Any +from typing import Any, ClassVar import numpy as np @@ -24,7 +24,7 @@ class CacheReader(ReaderBase): cannot change the files observed by an active reader. """ - name = "cache" + name: ClassVar[str] = "cache" def __init__( self, diff --git a/src/spine/io/read/larcv.py b/src/spine/io/read/larcv.py index 995af09b..ec642b02 100644 --- a/src/spine/io/read/larcv.py +++ b/src/spine/io/read/larcv.py @@ -2,7 +2,7 @@ import os from collections.abc import Sequence -from typing import Any +from typing import Any, ClassVar import numpy as np @@ -33,7 +33,7 @@ class LArCVReader(ReaderBase): trees corresponding to each of the requested data products. """ - name: str = "larcv" + name: ClassVar[str] = "larcv" def __init__( self, diff --git a/src/spine/io/read/supera.py b/src/spine/io/read/supera.py new file mode 100644 index 00000000..b3270150 --- /dev/null +++ b/src/spine/io/read/supera.py @@ -0,0 +1,267 @@ +"""Projected event reads from the pysupera 4.1 HDF5 format.""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from importlib import import_module +from typing import Any, ClassVar, cast + +import h5py +import numpy as np + +from .base import ReaderBase +from .hdf5.common import require_group + +__all__ = ["SuperaReader"] + + +def _dataset(file: h5py.File | h5py.Group, key: str) -> h5py.Dataset: + """Retrieve a required dataset and reject a group at the same path. + + Parameters + ---------- + file : h5py.File or h5py.Group + Open input file or parent group. + key : str + Required dataset path. + + Returns + ------- + h5py.Dataset + Dataset handle owned by the open file. + + Raises + ------ + TypeError + If the path refers to a group rather than a dataset. + """ + value = file[key] + if not isinstance(value, h5py.Dataset): + raise TypeError(f"Expected a dataset at {key}.") + return value + + +class SuperaReader(ReaderBase): + """Read truth deposits and event-local particle ranges without loading hits. + + Products are ``points``, ``particles``, ``inter``, ``events``, ``metadata`` + and ``groups``. Hit labels require an explicit plane product such as + ``hit_labels/volume0/plane0``. No HDF5 handles survive a read, making the + reader safe to pickle or use in data-loader workers. + """ + + name: ClassVar[str] = "supera" + default_keys = ("points", "particles", "inter", "events", "metadata") + + def __init__( + self, + file_keys: str | list[str] | None = None, + file_list: str | None = None, + keys: Sequence[str] | None = None, + limit_num_files: int | None = None, + max_print_files: int = 10, + **selection: Any, + ) -> None: + """Resolve files, validate format and apply ReaderBase entry selection. + + Parameters + ---------- + file_keys : str or list[str], optional + Input paths or glob patterns. + file_list : str, optional + Text file containing input paths. + keys : sequence[str], optional + Raw products to read; defaults exclude groups and hit labels. + limit_num_files : int, optional + Maximum number of input files. + max_print_files : int, default 10 + Maximum number of paths printed while resolving files. + **selection : dict + Entry selection arguments accepted by ReaderBase.process_entry_list. + Run/event selection is unavailable because pysupera does not define + run/subrun identities. + """ + # Register optional compression filters used by upstream productions. + try: + import_module("hdf5plugin") + except ImportError: + pass + + # Inspect only format metadata and offsets while building the event index. + self.process_file_paths(file_keys, file_list, limit_num_files, max_print_files) + self.keys = tuple(self.default_keys if keys is None else keys) + counts = [self._validate_file(path) for path in self.file_paths] + + # ReaderBase maps selected global entries back to their physical files. + self.num_entries = sum(counts) + self.file_offsets = np.r_[0, np.cumsum(counts)[:-1]].astype(np.int64) + self.file_index = np.repeat(np.arange(len(counts)), counts) + self.process_entry_list(**selection) + + def _validate_file(self, path: str) -> int: + """Check format metadata without materializing event payloads. + + Parameters + ---------- + path : str + Input HDF5 path. + + Returns + ------- + int + Number of events in the validated file. + + Raises + ------ + TypeError + If the format, offsets or required HDF5 object types are invalid. + ValueError + If the point layout or requested products are invalid. + """ + # Pylint infers Group for h5py lookups even after _dataset validates them. + # pylint: disable=no-member + with h5py.File(path, "r") as file: + if ( + "format_version" not in file + or _dataset(file, "format_version").asstr()[()] != "4.1.0" + ): + raise TypeError(f"{path}: expected pysupera format 4.1.0.") + + # Each table has its own event offsets and row domain. + num_entries = int(_dataset(file, "n_events")[()]) + for offset, target in ( + ("events/offsets", "particles/id"), + ("points/offsets", "points/flat"), + ("inter/offsets", "inter/interaction_id"), + ): + offsets = _dataset(file, offset)[:] + if ( + offsets.shape != (num_entries + 1,) + or offsets[0] != 0 + or np.any(np.diff(offsets) < 0) + or offsets[-1] != len(_dataset(file, target)) + ): + raise TypeError(f"{path}: invalid {offset}.") + + # Confirm column names and physical width before parsing point data. + columns = tuple(_dataset(file, "points/columns").asstr()[:]) + if ( + columns[:3] != ("x", "y", "z") + or "dE" not in columns + or not {"t", "time"}.intersection(columns) + or _dataset(file, "points/flat").ndim != 2 + or _dataset(file, "points/flat").shape[1] != len(columns) + ): + raise ValueError(f"{path}: unsupported point columns {columns}.") + + # Large hit-label arrays are opt-in through an explicit plane product. + for key in self.keys: + if key not in self.default_keys + ("groups",) and not ( + key.startswith("hit_labels/") + and key.count("/") == 2 + and key in file + and isinstance(file[key], h5py.Group) + and "offsets" in require_group(file, key) + ): + raise ValueError(f"Unsupported Supera product: {key}") + return num_entries + + def get(self, idx: int) -> dict[str, Any]: + """Read selected products for one event, rebasing absolute ranges. + + Parameters + ---------- + idx : int + Index in the selected entry list. + + Returns + ------- + dict[str, Any] + Projected products and standard source-file provenance. Point and + particle ranges are local to the returned event. + """ + # Dataset members are validated by _dataset; h5py lookup inference is dynamic. + # pylint: disable=no-member + file_index = self.get_file_index(idx) + entry_index = self.get_file_entry_index(idx) + result: dict[str, Any] = { + "index": idx, + "file_index": file_index, + "file_entry_index": entry_index, + } + result.update(self.get_source_provenance(file_index, entry_index)) + + # Open per read so worker processes never inherit live HDF5 handles. + with h5py.File(self.file_paths[file_index], "r") as file: + point_start, point_stop = _dataset(file, "points/offsets")[ + entry_index : entry_index + 2 + ] + + # Hit labels require an explicit plane path; never default to all hits. + for key in self.keys: + if key == "metadata": + result[key] = dict(file.attrs) + result[key]["columns"] = tuple( + _dataset(file, "points/columns").asstr()[:] + ) + elif key == "points": + result[key] = _dataset(file, "points/flat")[point_start:point_stop] + elif key == "events": + result[key] = { + name: _dataset(file, f"{key}/{name}")[entry_index] + for name in require_group(file, key) + if name != "offsets" + } + else: + # Each physical table has its own event offsets. + result[key] = self._read_table(file, key, entry_index) + return result + + @staticmethod + def _read_table( + file: h5py.File, key: str, entry_index: int + ) -> dict[str, np.ndarray]: + """Slice a columnar product and rebase its stored ranges. + + Parameters + ---------- + file : h5py.File + Open input file. + key : str + Particle, interaction, group or explicit hit-plane product path. + entry_index : int + Event index within this file. + + Returns + ------- + dict[str, np.ndarray] + Event-local columns. Hit and deposit-group labels retain their own + row domains and are never assumed to align with points. + """ + group = require_group(file, key) + offset_key = "events/offsets" if key == "particles" else f"{key}/offsets" + start, stop = _dataset(file, offset_key)[entry_index : entry_index + 2] + point_start = _dataset(file, "points/offsets")[entry_index] + particle_start = _dataset(file, "events/offsets")[entry_index] + + # Slice each column before performing any range arithmetic. + table: dict[str, np.ndarray] = {} + # HDF5 groups yield child names; their iterator typing is overly broad. + for name in cast(Iterable[str], group): + if name in ("offsets", "volume_offsets", "volume_offsets_index"): + continue + values = _dataset(group, name)[start:stop] + if key in ("particles", "inter"): + if "pc_" in name: + values = np.where(values >= 0, values - point_start, values) + elif key == "inter" and name in ("part_start", "part_end"): + values = np.where(values >= 0, values - particle_start, values) + table[name] = values + + # Volume starts index the event's deposit-group array, not its points. + if key == "groups": + start, stop = _dataset(group, "volume_offsets_index")[ + entry_index : entry_index + 2 + ] + table["volume_offsets"] = _dataset(group, "volume_offsets")[start:stop] + return table diff --git a/test/test_io/test_supera/coverage.ini b/test/test_io/test_supera/coverage.ini new file mode 100644 index 00000000..eb186c9a --- /dev/null +++ b/test/test_io/test_supera/coverage.ini @@ -0,0 +1,13 @@ +# Strict coverage for native pysupera input support, including error paths. +[run] +branch = True +source = src/spine + +[report] +include = + src/spine/io/read/supera.py + src/spine/io/dataset/supera.py + src/spine/io/parse/supera/* +exclude_lines = +fail_under = 100 +show_missing = True diff --git a/test/test_io/test_supera/test_supera.py b/test/test_io/test_supera/test_supera.py new file mode 100644 index 00000000..87ec951f --- /dev/null +++ b/test/test_io/test_supera/test_supera.py @@ -0,0 +1,402 @@ +"""Native pysupera reader and aligned supervision contracts.""" + +import copy +import pickle + +import h5py +import numpy as np +import pytest + +from spine.io.collate import CollateAll +from spine.io.dataset import SuperaDataset +from spine.io.parse.supera import SuperaTensorParser +from spine.io.read import SuperaReader + +SCHEMA = { + key: {"provider": provider} + for key, provider in ( + ("data", "tensor"), + ("seg_label", "segmentation"), + ("clust_label", "cluster_label"), + ("ppn_label", "particle_points"), + ("particles", "particle"), + ) +} + + +@pytest.fixture +def sample(tmp_path): + """Two events with sparse IDs, overlap, LE in a track and empty truth.""" + path = tmp_path / "supera.h5" + ids = np.array([10, 80, 99, 120, 150], dtype=np.int32) + # Track, shower, LE fragment of track, delta, Michel. + table = dict( + id=ids, + parent_id=ids, + ancestor_id=ids, + frag_id=ids, + inst_id=np.array([10, 80, -1, 120, 150]), + frag_inst_id=np.array([10, 80, 10, 120, 150]), + sem_type=np.array([2, 1, 5, 3, 4]), + frag_sem_type=np.array([2, 1, 5, 3, 4]), + inst_sem_type=np.array([2, 1, -1, 3, 4]), + frag_pc_start=np.array([0, 3, 2, 4, 5]), + frag_pc_end=np.array([2, 4, 3, 5, 6]), + inst_pc_start=np.array([0, 3, -1, 4, 5]), + inst_pc_le_start=np.array([2, 4, -1, 5, 6]), + inst_pc_end=np.array([3, 4, -1, 5, 6]), + pdg=np.array([13, 11, 11, 11, 11]), + parent_pdg=np.array([13] * 5), + geant4_trackid=ids, + interaction_id=np.zeros(5, dtype=int), + mass=np.ones(5), + ) + for key in ( + "start_pos", + "end_pos", + "first_pos", + "last_pos", + "start_mom", + "end_mom", + ): + table[key] = np.tile([1.5, 1.5, 1.5, 10.0], (5, 1)) + table["last_pos"][0, :3] = [4.5, 1.5, 1.5] + table["first_pos"][3, :3] = [10.5, 1.5, 1.5] + table["first_pos"][4, :3] = [13.5, 1.5, 1.5] + table["last_pos"][1] = np.nan + points = np.array( + [ + [1.5, 1.5, 1.5, 5, 2, 0], + [4.5, 1.5, 1.5, 2, 3, 0], + [7.5, 1.5, 1.5, 1, 4, 0], + [1.5, 1.5, 1.5, 0, 9, 0], + [10.5, 1.5, 1.5, 1, 5, 0], + [13.5, 1.5, 1.5, 1, 6, 0], + ], + dtype=np.float32, + ) + with h5py.File(path, "w") as file: + file["format_version"] = "4.1.0" + file["n_events"] = 3 + file.attrs["voxel_size"] = "3.0" + file.attrs["hit_energy"] = "charge" + file["events/offsets"] = [0, 5, 10, 10] + file["events/event_id"] = [11, 12, 13] + file["points/offsets"] = [0, 6, 12, 12] + file["points/flat"] = np.concatenate([points, points]) + file["points/columns"] = np.array( + ["x", "y", "z", "t", "dE", "dX"], dtype=h5py.string_dtype() + ) + for key, val in table.items(): + second = np.where(val >= 0, val + 6, val) if "pc_" in key else val + file["particles/" + key] = np.concatenate([val, second]) + file["inter/offsets"] = [0, 1, 2, 2] + file["inter/interaction_id"] = [0, 0] + file["inter/part_start"] = [0, 5] + file["inter/part_end"] = [5, 10] + file["inter/pc_start"] = [0, 6] + file["inter/pc_end"] = [6, 12] + file["groups/offsets"] = [0, 9, 18, 18] + file["groups/fragment_id"] = [10] * 18 + file["groups/is_le"] = [False] * 18 + file["groups/volume_offsets_index"] = [0, 2, 4, 6] + file["groups/volume_offsets"] = [0, 4, 0, 4, 0, 0] + # A huge virtual dataset: a projected read must never access it. + hit_plane = file.create_group("hit_labels/volume0/plane0") + hit_plane["offsets"] = [0, 2, 4, 4] + hit_plane.create_dataset( + "fragment_id", shape=(10**9,), dtype="i4", fillvalue=10 + ) + return str(path) + + +def dataset(path, **kwargs): + """Build the complete parser schema for one synthetic input file.""" + return SuperaDataset(file_keys=path, dtype="float32", schema=SCHEMA, **kwargs) + + +def test_reader_projection_rebasing_multifile_and_pickle(sample, monkeypatch): + """Preserve projection, event selection and range bases across worker copies.""" + original = h5py.Dataset.__getitem__ + + def guarded(data_source, key): + """Fail if default training reads touch group or hit-label payloads.""" + assert not data_source.name.startswith("/hit_labels/") + assert not data_source.name.startswith("/groups/") + return original(data_source, key) + + monkeypatch.setattr(h5py.Dataset, "__getitem__", guarded) + reader = SuperaReader(file_keys=[sample, sample], entry_list=[4, 0, 2]) + first, second, empty = pickle.loads(pickle.dumps(reader)).get_many([0, 1, 2]) + assert first["file_index"] == 1 and first["file_entry_index"] == 1 + np.testing.assert_array_equal(first["points"], second["points"]) + np.testing.assert_array_equal( + first["particles"]["inst_pc_start"], second["particles"]["inst_pc_start"] + ) + assert first["inter"]["part_start"][0] == 0 + assert empty["points"].shape == (0, 6) + assert "groups" not in first and "hit_labels" not in first + + +def test_explicit_group_and_hit_slices(sample): + """Read only the selected event from independent group and hit domains.""" + reader = SuperaReader( + file_keys=sample, keys=["groups", "hit_labels/volume0/plane0"] + ) + event = reader[1] + assert len(event["groups"]["fragment_id"]) == 9 + assert len(event["hit_labels/volume0/plane0"]["fragment_id"]) == 2 + np.testing.assert_array_equal(event["groups"]["volume_offsets"], [0, 4]) + + +def test_alignment_energy_semantics_and_endpoints(sample): + """Keep energy, semantic, fragment and endpoint supervision aligned.""" + event = dataset(sample)[1] + np.testing.assert_array_equal(event["data"].features[:, 0], [11, 3, 4, 5, 6]) + np.testing.assert_array_equal(event["seg_label"].features[:, 0], [1, 1, 4, 3, 2]) + np.testing.assert_array_equal(event["clust_label"].cluster_ids, [0, 0, 2, 3, 4]) + np.testing.assert_array_equal(event["clust_label"].group_ids, [0, 0, 0, 3, 4]) + np.testing.assert_array_equal( + event["clust_label"].shapes, event["seg_label"].features[:, 0] + ) + np.testing.assert_array_equal( + event["data"].coordinate_data, event["seg_label"].coordinate_data + ) + np.testing.assert_array_equal( + event["data"].coordinate_data, event["clust_label"].coords + ) + np.testing.assert_allclose(event["data"].meta.size, 0.3) + ppn = event["ppn_label"] + assert len(ppn.features) == 4 # Track start/end, delta start, Michel start. + np.testing.assert_array_equal(ppn.features[:, 0], [1, 1, 3, 2]) + np.testing.assert_array_equal(ppn.features[:, 2], [0, 1, 0, 0]) + assert np.isfinite(ppn.coordinate_data).all() + + +def test_empty_event_and_collation(sample): + """Collate empty events without changing event-local particle references.""" + data_source = dataset(sample) + empty = data_source[2] + assert empty["data"].features.shape == (0, 1) + assert empty["ppn_label"].features.shape == (0, 3) + batch = CollateAll(data_keys=SCHEMA)([data_source[0], empty, data_source[1]]) + np.testing.assert_array_equal(batch["data"].counts, [5, 0, 5]) + np.testing.assert_array_equal( + batch["clust_label"].shapes.tensor, batch["seg_label"].features.tensor[:, 0] + ) + assert batch["clust_label"].particle_indexes.tensor[-1] == 4 + assert batch["ppn_label"].features.tensor[-1, 1] == 4 + + +def test_invalid_version_offsets_ranges_and_grid(sample): + """Reject incompatible files, corrupt ranges and inconsistent parser grids.""" + raw = SuperaReader(file_keys=sample)[0] + raw["particles"]["frag_pc_end"][0] = 4 + with pytest.raises(ValueError, match="overlapping"): + SuperaTensorParser("float32")(raw) + schema = copy.deepcopy(SCHEMA) + schema["data"]["voxel_size_mm"] = 6 + with pytest.raises(ValueError, match="same voxel"): + SuperaDataset(file_keys=sample, dtype="float32", schema=schema) + with h5py.File(sample, "r+") as file: + file["points/offsets"][1] = 100 + with pytest.raises(TypeError, match="offsets"): + SuperaReader(file_keys=sample) + with h5py.File(sample, "r+") as file: + file["format_version"][()] = "3.0.0" + with pytest.raises(TypeError, match="4.1.0"): + SuperaReader(file_keys=sample) + + +def test_winner_uses_time_then_source_order(sample): + """Resolve equal-class overlaps deterministically without losing energy.""" + raw = SuperaReader(file_keys=sample)[0] + # Make the shower a second track in the same cell. Earlier time wins. + raw["particles"]["inst_sem_type"][1] = 2 + from spine.io.parse.supera import SuperaClusterLabelParser + + parser = SuperaClusterLabelParser("float32") + event = parser(raw) + assert event.cluster_ids[0] == 1 + assert event.values[0] == 11 + raw.pop("_supera_prepared") + raw["points"][3, 3] = raw["points"][0, 3] + assert parser(raw).cluster_ids[0] == 0 + + +def test_unknown_and_dangling_relationships_rejected(sample): + """Reject unsupported semantics and particle links with no target row.""" + raw = SuperaReader(file_keys=sample)[0] + raw["particles"]["inst_sem_type"][0] = 6 + with pytest.raises(ValueError, match="Unknown Supera semantic"): + SuperaTensorParser("float32")(raw) + raw = SuperaReader(file_keys=sample)[0] + raw["particles"]["frag_inst_id"][0] = 999 + with pytest.raises(ValueError, match="Dangling"): + SuperaTensorParser("float32")(raw) + + +def test_grid_transform_and_particle_energy(sample): + """Preserve detector transforms and distinguish source from merged energy.""" + schema = copy.deepcopy(SCHEMA) + for parser in schema.values(): + parser["voxel_size_mm"] = [3, 3, 3] + parser["origin_mm"] = [9, -12, 3] + event = SuperaDataset(file_keys=sample, dtype="float32", schema=schema)[0] + np.testing.assert_allclose(event["ppn_label"].coordinate_data[0], [0.5, 0.5, 0.5]) + np.testing.assert_allclose(event["data"].meta.lower, [0, 0, 0]) + # The winning track sees 11 MeV in the overlapped input cell, but its + # source-instance truth includes only its own 2 + 3 + 4 MeV contributions. + assert event["particles"][0].energy_deposit == 9 + + +def test_registered_factories(sample): + """Register native Supera readers and datasets under the same provider.""" + from spine.io.factories import dataset_factory, reader_factory + + assert isinstance( + reader_factory({"provider": "supera", "file_keys": sample}), SuperaReader + ) + data_source = dataset_factory( + {"provider": "supera", "file_keys": sample, "schema": SCHEMA}, dtype="float32" + ) + assert isinstance(data_source, SuperaDataset) + assert len(data_source) == 3 + + +@pytest.mark.parametrize("origin", [(0, 0), (0, np.nan, 0)]) +def test_reject_invalid_origin(origin): + """Reject malformed or nonfinite grid origins before reading events.""" + with pytest.raises(ValueError, match="origin_mm"): + SuperaTensorParser("float32", origin_mm=origin) + + +@pytest.mark.parametrize("size", [0, -1, np.inf]) +def test_reject_invalid_explicit_voxel_size(size): + """Reject invalid explicit grid dimensions at parser construction.""" + with pytest.raises(ValueError, match="voxel_size_mm"): + SuperaTensorParser("float32", voxel_size_mm=size) + + +@pytest.mark.parametrize("size", [None, "0", "[3, -1, 3]", np.nan]) +def test_reject_invalid_file_voxel_size(sample, size): + """Validate missing, encoded and numeric voxel-size attributes.""" + raw = SuperaReader(file_keys=sample)[0] + raw["metadata"]["voxel_size"] = size + with pytest.raises(ValueError, match="voxel_size|voxel size"): + SuperaTensorParser("float32")(raw) + + +@pytest.mark.parametrize( + ("column", "row", "value", "message"), + [ + ("id", 1, 10, "unique"), + ("id", 1, -1, "nonnegative"), + ("inst_pc_le_start", 0, 4, "LE range"), + ("frag_pc_end", 0, 1, "cover every point"), + ("frag_inst_id", 0, 80, "ownership"), + ("frag_sem_type", 2, 2, "semantics"), + ], +) +def test_reject_inconsistent_particle_table(sample, column, row, value, message): + """Reject corrupt identity, range and cross-level association fields.""" + raw = SuperaReader(file_keys=sample)[0] + raw["particles"][column][row] = value + with pytest.raises(ValueError, match=message): + SuperaTensorParser("float32")(raw) + + +@pytest.mark.parametrize("column", [0, 3, 4]) +def test_reject_nonfinite_point_values(sample, column): + """Reject nonfinite coordinates, times and deposited energy.""" + raw = SuperaReader(file_keys=sample)[0] + raw["points"][0, column] = np.nan + with pytest.raises(ValueError, match="Nonfinite"): + SuperaTensorParser("float32")(raw) + + +def test_empty_representative_retains_particle_row(sample): + """Keep truth rows for a representative whose point ranges are absent.""" + from spine.io.parse.supera import SuperaParticleParser + + raw = SuperaReader(file_keys=sample)[0] + raw["points"] = raw["points"][:5] + for key in ( + "frag_pc_start", + "frag_pc_end", + "inst_pc_start", + "inst_pc_end", + "inst_pc_le_start", + ): + raw["particles"][key][-1] = -1 + particles = SuperaParticleParser("float32")(raw) + assert len(particles) == 5 + assert particles[-1].num_voxels == 0 + assert particles[-1].energy_deposit == 0 + assert particles[-1].pdg_code == 11 + + +@pytest.mark.parametrize("kind", ["le", "nucleus", "nan", "below", "above"]) +def test_ppn_omits_ineligible_endpoints(sample, kind): + """Omit LE, nuclear, nonfinite and out-of-image instance targets.""" + from spine.io.parse.supera import SuperaParticlePointsParser + + raw = SuperaReader(file_keys=sample)[0] + table = raw["particles"] + if kind == "le": + table["inst_sem_type"][3] = 5 + table["frag_sem_type"][3] = 5 + table["inst_pc_le_start"][3] = table["inst_pc_start"][3] + elif kind == "nucleus": + table["pdg"][3] = 1000020040 + else: + table["first_pos"][3, 0] = {"nan": np.nan, "below": -1, "above": 100}[kind] + points = SuperaParticlePointsParser("float32")(raw) + assert len(points.features) == 3 + assert 3 not in points.features[:, 1] + + +def test_reader_without_compression_plugin(sample, monkeypatch): + """Uncompressed input remains usable without the optional filter package.""" + from spine.io.read import supera + + def missing_plugin(name): + """Simulate an absent optional dependency without changing other imports.""" + assert name == "hdf5plugin" + raise ImportError(name) + + monkeypatch.setattr(supera, "import_module", missing_plugin) + assert len(SuperaReader(file_keys=sample)[0]["points"]) == 6 + + +@pytest.mark.parametrize("key", ["unknown", "hit_labels", "hit_labels/volume0/plane9"]) +def test_reject_unknown_projection(sample, key): + """Require valid raw product names and explicit existing hit planes.""" + with pytest.raises(ValueError, match="Unsupported Supera product"): + SuperaReader(file_keys=sample, keys=[key]) + + +def test_reject_invalid_point_columns(sample): + """Validate declared point columns before any parser can misinterpret them.""" + with h5py.File(sample, "r+") as file: + file["points/columns"][0] = "wrong" + with pytest.raises(ValueError, match="unsupported point columns"): + SuperaReader(file_keys=sample) + + +def test_reject_group_in_place_of_dataset(sample): + """Reject malformed HDF5 structure with a clear dataset diagnostic.""" + with h5py.File(sample, "r+") as file: + del file["points/columns"] + file.create_group("points/columns") + with pytest.raises(TypeError, match="Expected a dataset"): + SuperaReader(file_keys=sample) + + +def test_raw_dataset_projection(sample): + """Expose raw selected products when no parser schema is supplied.""" + data = SuperaDataset(file_keys=sample, keys=["events"])[1] + assert data["events"]["event_id"] == 12 + assert "points" not in data + data = SuperaDataset(file_keys=sample)[0] + assert set(SuperaReader.default_keys).issubset(data) diff --git a/test/test_io/test_supera/test_supera_model.py b/test/test_io/test_supera/test_supera_model.py new file mode 100644 index 00000000..36581b47 --- /dev/null +++ b/test/test_io/test_supera/test_supera_model.py @@ -0,0 +1,117 @@ +"""Opt-in real-file training smoke test using models available on main. + +Run with SPINE_SUPERA_SAMPLE pointing at a pysupera 4.1 production file. +""" + +import os + +import numpy as np +import pytest + +SAMPLE = os.environ.get("SPINE_SUPERA_SAMPLE") +pytestmark = pytest.mark.skipif( + not SAMPLE, reason="Set SPINE_SUPERA_SAMPLE for real-file training" +) + + +def test_real_supera_forward_backward(): + """Backpropagate every training objective through the complete real event.""" + torch = pytest.importorskip("torch") + pytest.importorskip("MinkowskiEngine") + pytest.importorskip("torch_cluster") + from spine.io.collate import CollateAll + from spine.io.dataset import SuperaDataset + from spine.model.graph_spice import GraphSPICE, GraphSPICELoss + from spine.model.uresnet.ppn import UResNetPPN, UResNetPPNLoss + + torch.manual_seed(7) + schema = { + key: {"provider": provider} + for key, provider in ( + ("data", "tensor"), + ("seg_label", "segmentation"), + ("clust_label", "cluster_label"), + ("ppn_label", "particle_points"), + ) + } + ds = SuperaDataset(file_keys=SAMPLE, dtype="float32", schema=schema, n_entry=1) + e = ds[0] + np.testing.assert_array_equal( + e["data"].coordinate_data, e["seg_label"].coordinate_data + ) + np.testing.assert_array_equal(e["data"].coordinate_data, e["clust_label"].coords) + np.testing.assert_array_equal( + e["seg_label"].features[:, 0], e["clust_label"].shapes + ) + # Keep the full real event, including all overlaps and endpoint targets. + batch = CollateAll(data_keys=schema)([e]) + batch = {k: v.to_tensor(dtype=torch.float32) for k, v in batch.items()} + size = int(e["data"].meta.count.max()) + 2 + backbone = { + "num_classes": 5, + "num_input": 1, + "filters": 4, + "depth": 2, + "reps": 1, + "spatial_size": size, + "activation": "relu", + "norm_layer": "none", + } + + # Segmentation and PPN share the standard UResNet backbone on main. + model = UResNetPPN(backbone, ppn={}).train() + objective = UResNetPPNLoss(backbone, uresnet_loss={}, ppn={}, ppn_loss={}) + metrics = objective(batch["seg_label"], batch["ppn_label"], **model(batch["data"])) + parameter = next(model.uresnet.backbone.parameters()) + report = {} + for name in ("uresnet_loss", "ppn_loss"): + assert torch.isfinite(metrics[name]) + grad = torch.autograd.grad(metrics[name], parameter, retain_graph=True)[0] + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + report[name] = float(metrics[name].detach()) + + # Graph-SPICE uses the same aligned voxel labels with its own mainline backbone. + graph_config = { + "use_raw_features": True, + "embedder": { + "coord_conv": False, + "uresnet": { + key: value for key, value in backbone.items() if key != "num_classes" + }, + }, + "kernel": {"provider": "bilinear", "num_features": 4}, + "constructor": { + "graph": {"provider": "radius", "r": 1.9}, + "min_size": 1, + "edge_threshold": 0.5, + }, + } + graph = GraphSPICE(graph_config).train() + graph_objective = GraphSPICELoss(graph_config, {"provider": "edge", "metric": None}) + graph_metrics = graph_objective( + batch["seg_label"], + batch["clust_label"], + **graph(batch["data"], batch["seg_label"]), + ) + assert torch.isfinite(graph_metrics["loss"]) + graph_parameter = next(graph.embedder.backbone.parameters()) + grad = torch.autograd.grad( + graph_metrics["loss"], graph_parameter, retain_graph=True + )[0] + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + report["graph_spice_loss"] = float(graph_metrics["loss"].detach()) + + total = metrics["loss"] + graph_metrics["loss"] + total.backward() + assert any( + p.grad is not None and p.grad.abs().sum() > 0 + for p in graph.kernel_fn.parameters() + ) + print( + { + "voxels": len(e["data"].features), + "ppn_targets": len(e["ppn_label"].features), + "losses": report, + "total_loss": float(total.detach()), + } + ) From efb09667ff583673a1b988b843f59d1f0ace91fc Mon Sep 17 00:00:00 2001 From: Francois Drielsma Date: Tue, 3 Nov 2026 14:05:20 -0800 Subject: [PATCH 2/5] Extend Supera input through generic training and truth conversion --- config/io/supera.yaml | 13 +- docs/source/api/io.rst | 146 ++++++- docs/source/api/supera_format_findings.rst | 226 ++++++++++ src/spine/io/parse/supera/__init__.py | 5 +- .../io/parse/supera/{truth.py => base.py} | 387 ++++++------------ src/spine/io/parse/supera/cluster.py | 45 ++ src/spine/io/parse/supera/metadata.py | 96 +++++ src/spine/io/parse/supera/particle.py | 282 +++++++++++++ src/spine/io/parse/supera/tensor.py | 65 +++ test/test_io/test_supera/__init__.py | 0 test/test_io/test_supera/conftest.py | 152 +++++++ test/test_io/test_supera/test_supera.py | 93 +---- test/test_io/test_supera/test_supera_chain.py | 156 +++++++ test/test_io/test_supera/test_supera_model.py | 34 +- test/test_io/test_supera/test_supera_truth.py | 331 +++++++++++++++ 15 files changed, 1658 insertions(+), 373 deletions(-) create mode 100644 docs/source/api/supera_format_findings.rst rename src/spine/io/parse/supera/{truth.py => base.py} (60%) create mode 100644 src/spine/io/parse/supera/cluster.py create mode 100644 src/spine/io/parse/supera/metadata.py create mode 100644 src/spine/io/parse/supera/particle.py create mode 100644 src/spine/io/parse/supera/tensor.py create mode 100644 test/test_io/test_supera/__init__.py create mode 100644 test/test_io/test_supera/conftest.py create mode 100644 test/test_io/test_supera/test_supera_chain.py create mode 100644 test/test_io/test_supera/test_supera_truth.py diff --git a/config/io/supera.yaml b/config/io/supera.yaml index 13c27459..51a44fb6 100644 --- a/config/io/supera.yaml +++ b/config/io/supera.yaml @@ -1,4 +1,5 @@ -# Native pysupera 4.1 truth-deposit input for segmentation, PPN and fragmentation. +# Native pysupera 4.1 input for generic-chain training and truth conversion. +# data contains truth-deposit energy (MeV), not measured pixel charge. # Merge this io block into a training configuration; no conversion is required. io: loader: @@ -17,7 +18,17 @@ io: provider: cluster_label ppn_label: provider: particle_points + coord_label: + provider: particle_coords + graph_label: + provider: particle_graph particles: provider: particle + meta: + provider: meta + source_ids: + provider: source_ids + event_info: + provider: event_info collate_fn: provider: all diff --git a/docs/source/api/io.rst b/docs/source/api/io.rst index 52302f14..289a4f48 100644 --- a/docs/source/api/io.rst +++ b/docs/source/api/io.rst @@ -70,7 +70,12 @@ productions. No pysupera or LArCV runtime is required. seg_label: {provider: segmentation} clust_label: {provider: cluster_label} ppn_label: {provider: particle_points} + coord_label: {provider: particle_coords} + graph_label: {provider: particle_graph} particles: {provider: particle} + meta: {provider: meta} + source_ids: {provider: source_ids} + event_info: {provider: event_info} collate_fn: {provider: all} ``SuperaDataset`` reuses the HDF5 dataset's parsing, collation and augmentation @@ -79,9 +84,10 @@ multiple files, ordinary entry selection, source-file provenance and projected reads. Its default raw products are ``points``, ``particles``, ``inter``, ``events`` and ``metadata``. Particle and interaction point ranges are rebased to the event's point array. Interaction particle ranges are likewise rebased. -Event identities remain available in the raw ``events`` product, including -production ``event_id`` and ``job_id`` when present; these are not invented -run/subrun numbers. +The ``event_info`` parser preserves every stored event/input/job identity as a +named integer tensor, including ``event_id``, ``input``, ``input_event``, +``jaxtpc_entry``, ``job_event`` and ``job_id`` in the sample. It does not invent +run/subrun numbers. The raw identities remain available as ``events``. Physical semantics ~~~~~~~~~~~~~~~~~~ @@ -107,8 +113,9 @@ in particular ``_run.py``, ``layout.py``, ``io_v3.py``, ``provenance.py`` and its instance. The reader never assumes group/point alignment. * The writer retains sparse particle IDs. Despite the upstream reader module's row-index wording, these must be resolved through an ID-to-row map. SPINE - particle indexes, ancestor links and PPN particle references are event-local - table indexes; the cluster sidecar's ``particle`` field retains source IDs. + particle IDs, cluster-sidecar ``particle`` IDs, ancestor links, graph nodes + and PPN references are all event-local particle-table rows. ``source_ids`` + preserves the sparse source ID for each row, without shifting it in a batch. * Semantic codes map explicitly: pysupera shower/track/delta/Michel/LE ``1/2/3/4/5`` become SPINE ``0/1/3/2/4``. Non-LE points use the instance's semantic class; LE fragments retain LE even inside a track or shower. @@ -124,8 +131,9 @@ Aligned training products All parsers share one event preparation. Coordinates are rasterized with the file's ``voxel_size`` and an origin of zero, then shifted by an integer number of cells to make event coordinates nonnegative. ``Meta`` records the inverse -transform in cm. Each parser accepts ``voxel_size_mm`` (scalar or three-vector) -and ``origin_mm``; all parsers in a dataset must use the same grid settings. +transform in cm. Every spatial parser accepts ``voxel_size_mm`` (scalar or three-vector) +and ``origin_mm``; all spatial parsers in a dataset must use the same grid settings. +The identity-only ``event_info`` parser does not take grid arguments. An explicit size is required when the file does not supply one. These settings also permit rasterizing deposit-mode files. Upstream voxelization can use a per-particle origin, so stored centers are rasterized rather than assumed to @@ -139,12 +147,85 @@ fragment IDs and fragment-row associations, with instance IDs in ``group``. Unprovided neutrino identity is left at ``-1``. No energy-to-charge conversion is performed. -PPN targets use finite instance starts and track ends only, exclude LE, +``coord_label`` is a separate, row-aligned ``particle_coords`` product for +GrapPA. Its ``TensorData`` schema has ``start`` and ``end`` coordinate groups, +plus ``time`` (ns) and ``shape`` features. Every stored particle gets a row, +including particles with no retained voxels. Instance rows use stored collected +step endpoints. Where these are missing, fragment rows use earliest/latest +collected deposit positions; instance rows use their non-LE deposit range, +or all deposits when that range is empty. These fallback positions are derived +from voxelized deposits, **not exact Geant4 step endpoints**. Non-track ends +repeat their starts. Rows without usable endpoints retain NaN coordinates and +times. Deposited rows have finite fallback coordinates; no creation/destruction +position is substituted. The same endpoint resolution populates parsed particle +``first_step``/``last_step`` fields. + +PPN targets use finite stored instance starts and track ends only, exclude LE, unknown classes and nuclear fragments, and omit points outside the event -image. Targets refer to the complete event particle table. The particle parser -returns that table as SPINE ``Particle`` objects in voxel units, preserving -missing kinematics as NaN. Particle energy deposits sum the source instance's -contributions, independently of overlap winner selection. +image. They reference the complete event particle table but cannot replace +``coord_label`` for closest-primary selection or track orientation. + +The particle parser fills PID, group/interaction-primary flags and ancestry +consistently with cluster supervision. PID and interaction-primary labels are +propagated from each fragment's host instance, preventing daughter-electron +votes from changing an instance's PID. A group primary is its representative; +an interaction primary is a host whose redirected stored parent is itself. +This is the file's stored-root convention, not an independently inferred +generator-primary definition. Host-instance interaction IDs label absorbed +fragments, whose own source interaction may differ. Stored interaction vertices +supply a common ``ancestor_position`` and cluster ``vertex`` for all members. +When unavailable, the stored ancestor's creation position is used; missing +ancestry remains NaN rather than indexing row ``-1``. + +Individual ``pc_start:pc_end`` ranges do not cover all deposits in this file: +absorbed particles may have been omitted from the stored table. Fragment ranges +do partition the point cloud. SPINE assigns each fragment's summed dE once, +to its representative row, and zero to other rows. Truth particle construction +then sums these fragment contributions without double-counting. This energy +is independent of overlap-winner assignment; a truth object's voxel deposition +sum can differ from its attributed source-fragment energy. + +Truth conversion +~~~~~~~~~~~~~~~~ + +Use the complete schema above (also in ``config/io/supera.yaml``) with: + +.. code-block:: yaml + + base: + unwrap: true + build: + mode: truth + units: cm + fragments: true + particles: true + interactions: true + +No model is required for truth-only construction. ``meta`` exposes the shared +``Meta`` at top level, so collation, unwrapping and ``BuildManager`` can convert +voxel coordinates to detector centimetres. Voxel centers use the half-cell +shift; particle endpoints and interaction vertices retain their continuous +physical positions. ``graph_label`` supplies directed stored-parent edges; +truth building contracts invisible ancestors and restores parent/child links +between retained particles. Without this product, those output genealogy links +are not restored. The standard SPINE HDF5 V2 writer/reader preserves these +products and associations; load truth objects through ``BuildManager`` to +reattach their long-form point arrays. + +Unavailable neutrino identity, unknown PID and missing relationship IDs use +``-1``. Missing PDG codes default to SPINE's ``INVALID_PDG`` (0); the source +Geant4-parent PDG sentinel (-1) is retained when present. Absent creation +process names are empty strings. Missing positions, times and travel distance +are NaN. Neutrino interaction records, measured charge, generator-process +strings and exact endpoints of individual fragments cannot be recovered from +this file. The reader does not synthesize them. See the +:doc:`author-facing format report ` for measured format +limitations and proposed upstream changes. + +.. toctree:: + :hidden: + + supera_format_findings ``groups`` and ``hit_labels/volume0/plane0`` (or another explicit plane path) can be requested through ``SuperaReader(keys=...)``. They are never read by @@ -177,15 +258,33 @@ Run the real-file forward/backward test in the model runtime with: -e SPINE_SUPERA_SAMPLE=/workspace/supera_pixel_j000000-000004.h5 \ -v "$PWD:/workspace" -w /workspace \ ghcr.io/deeplearnphysics/spine:latest bash -lc \ - 'pip install hdf5plugin && pytest -q -s test/test_io/test_supera/test_supera_model.py' - -The first sample event yields 25,537 unique voxels and 120 PPN targets. The -integration test exercises segmentation and PPN through the standard -UResNet+PPN model and fragmentation through Graph-SPICE, using the full event. -It requires finite, nonzero backbone gradients for each objective and verifies -that the combined backward pass reaches the graph kernel. Both models are -available on ``main``; this input implementation has no dependency on the -separate UResNetMulti feature branch. + 'pip install hdf5plugin && pytest -q -s test/test_io/test_supera' + +The first sample event yields 25,537 unique voxels and 120 PPN targets. +The opt-in integration suite exercises these generic-chain components: + +* UResNet segmentation and PPN forward/backward on the complete first event, + with segmentation-stage cache publication and readback. +* Graph-SPICE fragmentation forward/backward on the same aligned voxel labels. +* Fragment-graph feature/target materialization and cache readback, followed by + shower GrapPA primary/grouping and track GrapPA grouping forward/backward. +* Particle-graph feature/target materialization and cache readback, followed by + interaction GrapPA PID, closest-primary, orientation and grouping + forward/backward. +* Truth fragment, particle and interaction construction for events 0 and 1, + followed by SPINE HDF5 V2 writing, reading and association rebuilding in cm. + +GrapPA checks use two complete events with truth-defined input clusters and the +maintained generic configuration, reducing only hidden neural widths and +enabling closest-primary matching. They retain all configured objectives and +assert valid, nonzero supervised counts and gradients for each. These checks +validate each learned component and its cache contract; they do not measure +reconstruction quality or assert that an untrained end-to-end network produces +useful clusters. Synthetic tests additionally exercise sparse IDs larger than +the particle table, empty batch entries, missing endpoints, missing ancestors, +closest-primary matching and opposite track orientations across events. +The Supera modules retain 100% line and branch coverage. All exercised models +are available on ``main``; there is no dependency on UResNetMulti. .. autosummary:: :toctree: generated @@ -195,6 +294,11 @@ separate UResNetMulti feature branch. parse.supera.SuperaClusterLabelParser parse.supera.SuperaParticleParser parse.supera.SuperaParticlePointsParser + parse.supera.SuperaParticleCoordsParser + parse.supera.SuperaParticleGraphParser + parse.supera.SuperaMetaParser + parse.supera.SuperaSourceIDsParser + parse.supera.SuperaEventInfoParser Entry Filtering --------------- diff --git a/docs/source/api/supera_format_findings.rst b/docs/source/api/supera_format_findings.rst new file mode 100644 index 00000000..57b7a023 --- /dev/null +++ b/docs/source/api/supera_format_findings.rst @@ -0,0 +1,226 @@ +pysupera 4.1 integration findings +================================= + +This report is prepared for the pysupera authors from the SPINE PR #191 +integration. It distinguishes an incorrect documented contract from intentional +loss of source detail and requests for a more explicit interchange schema. +It has not been submitted upstream. + +Reference production +-------------------- + +* File: ``supera_pixel_j000000-000004.h5``, 1,000 events, format ``4.1.0``. +* Embedded software identity: ``drinkingkazu/pysupera`` commit + ``dab92d8fc850c09d0ca85cff16c6f36c28cf6d4b``, clean working tree. +* Producer: ``doraemon_prod.stages.supera``, EDepSim plus JAXTPC pixel input. +* Configuration: 3 mm stored voxels, ``hit_energy=charge``, + ``hit_x_from=nominal``, ``kinematics_steps=visible``. + +The source reference is `the pinned upstream tree +`_. +Measurements below use event 0, reading only that event's particle, interaction +and point slices. No hit-label payload was loaded. + +1. Sparse IDs contradict the documented row-index contract +---------------------------------------------------------- + +**Confirmed documentation/contract mismatch.** ``io_v3.py`` describes particle +IDs as event-local indexes into ``events/offsets[ev] + id``. The file instead +has 5,896 particle rows, with source IDs spanning 0 through 14,152. +``layout.py`` writes the retained original particle IDs and redirects parent +links in that same source-ID namespace. Sparse IDs are not themselves a format +error; interpreting them as rows is. + +SPINE originally normalized some relationships but retained source IDs in its +cluster ``particle`` label. Truth building then failed with +``Invalid particle ID found in fragment labels.`` That mixed-namespace adapter +bug is fixed by mapping all internal relationships to particle-table rows and +preserving source IDs in a separate provenance product. + +Requested upstream change: + +* Specify the namespace of every ID column, including ``frag_parent_id``, + ``inst_parent_id`` and hit/group labels. Publish the negative sentinel. +* Either write normalized row indexes plus a separate ``source_id``, or retain + sparse IDs and correct the reader documentation and examples to require a + source-ID-to-row map. Do not change the meaning silently within version 4.1. +* Validate uniqueness and nonnegative-ID referential integrity per event. + +2. Own-particle ranges do not partition the stored deposits +----------------------------------------------------------- + +**Observed loss of per-source attribution, consistent with selective row +retention in** ``layout.py``. The event has 28,789 stored point rows. The union +of the stored particles' ``pc_start:pc_end`` ranges covers only 15,334 of them, +without overlaps. The remaining 13,455 points contribute 4,686.0472 MeV of the +11,938.2904 MeV total. Summing only stored own-particle ranges gives +7,252.2432 MeV. + +Fragment and instance ranges do partition all point rows. ``layout.py`` retains +visible representatives and ancestry, not every contributing source particle. +The missing own-particle coverage therefore does **not** establish that the +file has lost deposit energy or that its fragment ranges are corrupt. It means +that complete per-source-particle energy cannot be recovered by summing those +stored own-particle ranges. + +SPINE accounts for all energy by integrating each fragment range exactly once +and assigning that contribution to its representative. Truth groups sum those +fragment contributions. This is fragment-attributed energy, not an assertion +that the representative's own Geant4 track deposited all of it. Overlap-winner +voxel sums and source-fragment energies are separate quantities. + +Requested upstream change: + +* Explicitly document that own-particle ranges may leave gaps, while fragment + and instance ranges are complete partitions. +* Store an explicit per-fragment/per-instance ``energy_deposit`` with units and + attribution semantics, or guarantee its derivation from the ranges. +* If complete source-particle truth is intended, retain the omitted contributing + rows or add a deposit-to-original-track association. SPINE cannot reconstruct + that information from fragment totals. + +3. Fragment endpoints are absent; deriving them is approximate +-------------------------------------------------------------- + +**Missing information for fragment-level supervision, not a malformed array.** +There are 1,046 instance representatives, of which 815 have finite stored first +positions. No non-instance row has a finite ``first_pos``. After SPINE's common +grid and overlap selection, 270 non-LE fragments remain visible; 193 of these +lack stored first positions. + +``kinematics.py`` explicitly produces ``first_*``/``last_*`` only for instances. +They describe counted Geant4 step starts/ends, preferentially from particles +with non-LE membership. Showers intentionally have no last endpoint. +``start_*``/``end_*`` instead describe a track's creation/destruction and are +not interchangeable with the observed trajectory endpoints. + +SPINE needs one coordinate-label row per stored particle for primary matching +and orientation supervision. It retains stored collected-step endpoints when +available. Missing endpoints are derived from earliest/latest deposits in the +fragment, or the instance's non-LE range when applicable. These positions are +voxelized deposit locations: they cannot recover the exact original step +boundary or sub-voxel trajectory. Rows without usable deposits remain NaN; +non-track end labels repeat their start labels. + +Requested upstream change: + +* Provide collected-step endpoint coordinates/times for fragment representatives + as well as instances, with the same LE and visibility policy. +* Define validity independently for position, time and momentum. Record whether + a coordinate is a true step boundary, deposit midpoint or voxel center. +* State the intended policy for equal-time deposits and split tracks. An + explicitly versioned endpoint product would remove SPINE's approximation. + +4. Source interaction and host interaction can differ +----------------------------------------------------- + +**Two legitimate meanings currently share indirect representation.** +``layout.py`` explicitly permits particles' deposits to be absorbed into groups +from another interaction. Particle rows retain their own interaction; points +are ordered by their group's interaction. + +In event 0, 19 fragment rows have an ``interaction_id`` different from their +host instance. After overlap selection, 26 winning voxels have this mismatch. +Using the fragment row's own interaction for grouping supervision is therefore +incorrect for those voxels. SPINE uses the host instance's interaction for +cluster supervision and reconstructed truth-group membership. The stored +interaction vertex supplies a common vertex within that group. + +Requested upstream change: + +* Expose distinct ``source_interaction_id`` and ``instance_interaction_id`` + concepts, or document the required ``frag_inst_id`` join next to the field. +* State explicitly which interaction should supervise deposit/fragment/instance + grouping, and whether cross-interaction absorption is intended for training. +* Retain the original generator ``vertex_id`` and document its relation to the + event-local interaction table. + +5. Stored roots and true generator primaries need distinct definitions +---------------------------------------------------------------------- + +**Semantic ambiguity outside the retained genealogy.** ``parent_id`` points to +the nearest stored ancestor and becomes self when none is available; +``ancestor_id`` is the root of that redirected graph. Event 0 has 61 such roots +and 17 interactions. This graph is useful and walkable, but a stored root is +not, by definition alone, proof of generator-primary status. Real direct-parent +track/PDG columns are also available and should retain their separate meaning. + +SPINE preserves the stored genealogy, uses a host's self-parent status for +interaction-primary supervision, and propagates that label to its fragments. +Missing parents remain unknown. This convention should be reviewed by the +producer before treating it as an independent physics-primary definition. + +Requested upstream change: + +* Define and, preferably, store ``interaction_primary`` explicitly, including + the treatment of prompt decays and missing/unwritten ancestors. +* Distinguish redirected stored-parent links from original Geant4 parents. + Document self-root versus negative/missing sentinel behavior. +* Preserve full generator/neutrino truth if consumers are expected to recover + it. An interaction position alone cannot supply neutrino identity or missing + process information. + +6. Truth deposits, measured charge and association domains +---------------------------------------------------------- + +**Schema clarity request, not a discovered charge-conversion bug.** Despite +``hit_energy=charge``, the file's ``points/flat`` is truth-deposit dE in MeV, +not measured pixel charge. The measured hit payload remains in the separate +JAXTPC file. A model trained on this product is learning from truth-deposit +energy, not detector charge. Its learned weights are not thereby validated for +measured-charge inference. + +``groups`` has 54,863,490 rows over the production, versus 31,871,012 point +rows. The group domain is JAXTPC deposit groups concatenated over volumes, not +point rows. Hit labels occupy yet another, very large domain. Their offset +relationships must be explicit; equal-length or row-alignment assumptions are +unsafe. + +Requested upstream change: + +* Attach units, value kind (truth energy versus measured charge), coordinate + convention and row-domain metadata to each product, not only production + configuration attributes. +* If standalone detector-input training is intended, include the measured hit + payload or a stable source-file identity/entry/volume mapping, plus an + explicitly defined many-to-many hit-to-truth association. +* Publish a schema with per-product offsets, cardinality checks and examples + that read event slices rather than whole hit-label arrays. + +Minimal reproduction +-------------------- + +This checks the ID/range findings without loading whole-file point or hit data: + +.. code-block:: python + + import h5py + import hdf5plugin # Register the file's compression filters. + import numpy as np + + with h5py.File("supera_pixel_j000000-000004.h5", "r") as f: + p0, p1 = f["events/offsets"][:2] + x0, x1 = f["points/offsets"][:2] + p = {key: value[p0:p1] for key, value in f["particles"].items()} + points = f["points/flat"][x0:x1] + columns = list(f["points/columns"].asstr()[:]) + cover = np.zeros(len(points), dtype=np.int32) + for start, stop in zip(p["pc_start"], p["pc_end"]): + if start >= 0: + cover[start - x0:stop - x0] += 1 + energy = points[:, columns.index("dE")] + print("particle rows, max ID:", len(p["id"]), p["id"].max()) + print("own-range coverage:", np.unique(cover, return_counts=True)) + print("total MeV:", energy.sum(dtype=np.float64)) + print("unattributed MeV:", energy[cover == 0].sum(dtype=np.float64)) + +Suggested upstream acceptance checks +------------------------------------ + +Before freezing the format, validate a fixture containing sparse source IDs, +absorbed/unwritten contributors, cross-interaction absorption, a split track, +a shower, missing endpoints and an empty event. Check ID namespaces, complete +fragment/instance point coverage, energy conservation, endpoint validity and +source-versus-host interaction semantics. Round-trip the fixture through the +upstream reader and a small independent consumer. Changing these contracts +should increment the format version. diff --git a/src/spine/io/parse/supera/__init__.py b/src/spine/io/parse/supera/__init__.py index 5c2c4fbb..3833e0d2 100644 --- a/src/spine/io/parse/supera/__init__.py +++ b/src/spine/io/parse/supera/__init__.py @@ -1,3 +1,6 @@ """Parsers for native pysupera truth-deposit products.""" -from .truth import * +from .cluster import * +from .metadata import * +from .particle import * +from .tensor import * diff --git a/src/spine/io/parse/supera/truth.py b/src/spine/io/parse/supera/base.py similarity index 60% rename from src/spine/io/parse/supera/truth.py rename to src/spine/io/parse/supera/base.py index b7aa9b3c..f3488b4f 100644 --- a/src/spine/io/parse/supera/truth.py +++ b/src/spine/io/parse/supera/base.py @@ -8,18 +8,10 @@ import numpy as np from spine.constants import PDG_TO_PID -from spine.data import ClusterLabelData, Meta, ObjectListData, Particle, TensorData +from spine.data import Meta from ..base import ParserBase -__all__ = [ - "SuperaClusterLabelParser", - "SuperaParticleParser", - "SuperaParticlePointsParser", - "SuperaSegmentationParser", - "SuperaTensorParser", -] - def _shape(code: int | np.ndarray) -> np.ndarray: """Translate one-based pysupera classes to SPINE semantic labels. @@ -38,7 +30,7 @@ def _shape(code: int | np.ndarray) -> np.ndarray: return np.array([-1, 0, 1, 3, 2, 4, -1])[np.clip(code, 0, 6)] -class SuperaTensorParser(ParserBase): +class SuperaParserBase(ParserBase): """Voxelize truth deposits once, sharing winner selection across products. Coordinates and optional grid arguments are in mm; output metadata uses @@ -48,8 +40,6 @@ class SuperaTensorParser(ParserBase): then earliest time and source row order. Unknown point classes are rejected. """ - name = "tensor" - def __init__( self, dtype: str, @@ -72,6 +62,7 @@ def __init__( points_event="points", particle_event="particles", metadata_event="metadata", + interaction_event="inter", ) # Normalize grid settings so all parsers can share the event cache. @@ -140,9 +131,134 @@ def prepare(self, trees: dict[str, Any]) -> dict[str, Any]: columns=table, source_ids=source_ids, ) + event["labels"] = self._particle_labels(event, trees["inter"]) cache[self.grid] = event return event + @staticmethod + def _particle_labels( + event: dict[str, Any], interactions: dict[str, np.ndarray] + ) -> dict[str, np.ndarray]: + """Build shared particle truth using local IDs and interaction vertices. + + Parameters + ---------- + event : dict[str, Any] + Prepared voxel image and resolved particle columns. + interactions : dict[str, np.ndarray] + Event-local interaction table, with detector vertices in mm. + + Returns + ------- + dict[str, np.ndarray] + Particle labels shared by compact supervision and truth objects. + Missing ancestry yields NaN vertices unless an interaction is known. + """ + table = event["columns"] + host = table["frag_inst_id"] + valid = (host >= 0) & (host < len(table["id"])) + interaction_ids = table["interaction_id"].copy() + interaction_ids[valid] = table["interaction_id"][host[valid]] + vertices = np.full((len(table["id"]), 3), np.nan) + ancestors = table["ancestor_id"] + valid_ancestors = ancestors >= 0 + vertices[valid_ancestors] = table["start_pos"][ancestors[valid_ancestors], :3] + + # A generator interaction may contain several distinct stored roots. + # Use its common vertex instead of arbitrarily selecting one root. + if all(key in interactions for key in ("x", "y", "z")): + positions = np.column_stack([interactions[key] for key in ("x", "y", "z")]) + for row, interaction in enumerate(interactions["interaction_id"]): + if np.isfinite(positions[row]).all(): + vertices[interaction_ids == interaction] = positions[row] + + shapes = _shape(table["sem_type"]) + shapes[valid] = _shape(table["inst_sem_type"][host[valid]]) + shapes[table["frag_sem_type"] == 5] = 4 + + # PID and interaction-primary supervision describe the host instance, + # including its secondary fragments, just as in the LArCV adapter. + pids = np.full(len(table["id"]), -1, dtype=np.int64) + pids[valid] = [ + PDG_TO_PID.get(int(pdg), -1) for pdg in table["pdg"][host[valid]] + ] + primaries = np.full(len(table["id"]), -1, dtype=np.int64) + primaries[valid] = np.where( + table["parent_id"][host[valid]] >= 0, + table["parent_id"][host[valid]] == host[valid], + -1, + ) + + # All relationship fields use the same event-local particle namespace. + return { + "particle": table["id"].copy(), + "group": host.copy(), + "ancestor": table["ancestor_id"].copy(), + "interaction": interaction_ids, + "nu": np.full(len(table["id"]), -1), + "pid": pids, + "group_primary": np.where(valid, table["id"] == host, -1), + "interaction_primary": primaries, + "vertex": (vertices / 10.0 - event["meta"].lower) / event["meta"].size, + "momentum": np.linalg.norm(table["start_mom"][:, :3], axis=1), + "energy_init": table["start_mom"][:, 3].copy(), + "shape": shapes, + } + + def endpoints(self, trees: dict[str, Any]) -> tuple[np.ndarray, np.ndarray]: + """Resolve collected-step endpoints without substituting track creation. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + Start and end positions in mm, with time in ns in the last column. + Missing endpoints use fragment deposits where available, otherwise + remain NaN. Non-track ends equal their starts. + """ + event = self.prepare(trees) + if "endpoints" in event: + return event["endpoints"] + table = event["columns"] + starts = table["first_pos"].copy() + ends = table["last_pos"].copy() + time_col = next( + i + for i, key in enumerate(trees["metadata"]["columns"]) + if key in ("t", "time") + ) + + # Non-instance rows do not have upstream collected-step kinematics. + for row in range(len(table["id"])): + prefix = "inst" if table["inst_id"][row] == row else "frag" + owner = row if prefix == "inst" else int(table["frag_id"][row]) + if owner >= 0: + lower, upper = ( + table[prefix + "_pc_start"][owner], + table[prefix + "_pc_end"][owner], + ) + # Match upstream's preference for non-LE trajectory deposits. + if prefix == "inst" and table["inst_pc_le_start"][row] > lower: + upper = table["inst_pc_le_start"][row] + points = trees["points"][max(0, lower) : max(0, upper)] + if len(points): + points = points[np.argsort(points[:, time_col], kind="stable")] + for endpoint, point in zip((starts, ends), points[[0, -1]]): + if not np.isfinite(endpoint[row]).all(): + endpoint[row, :3] = point[:3] + endpoint[row, 3] = point[time_col] * 1000.0 + + # A shower has one meaningful collected-step endpoint. + ends[event["labels"]["shape"] != 1] = starts[event["labels"]["shape"] != 1] + starts[~np.isfinite(starts)] = np.nan + ends[~np.isfinite(ends)] = np.nan + event["endpoints"] = (starts, ends) + return starts, ends + def _voxelize( self, points: np.ndarray, @@ -338,250 +454,3 @@ def _point_labels( ): raise ValueError("Fragment semantics disagree with instance LE ranges.") return fragment, instance, semantic - - def __call__(self, trees: dict[str, Any]) -> TensorData: - """Return summed truth energy in MeV on the shared sparse grid. - - Parameters - ---------- - trees : dict[str, Any] - Raw event products returned by SuperaReader. - - Returns - ------- - TensorData - Sparse coordinates and summed deposit energy in MeV. - """ - # Reuse the same grid and overlap winners as the input tensor. - event = self.prepare(trees) - return TensorData( - coords=event["coords"].astype(self.itype), - features=event["energy"].astype(self.ftype)[:, None], - meta=event["meta"], - ) - - -class SuperaSegmentationParser(SuperaTensorParser): - """Return semantic labels in exactly the input voxel order.""" - - name = "segmentation" - - def __call__(self, trees: dict[str, Any]) -> TensorData: - """Return five-class labels; unknown point semantics are rejected. - - Parameters - ---------- - trees : dict[str, Any] - Raw event products returned by SuperaReader. - - Returns - ------- - TensorData - Semantic labels sharing the input tensor coordinates and row order. - """ - # Reuse the same grid and overlap winners as the input tensor. - event = self.prepare(trees) - return TensorData( - coords=event["coords"].astype(self.itype), - features=event["shape"].astype(self.ftype)[:, None], - meta=event["meta"], - ) - - -class SuperaClusterLabelParser(SuperaTensorParser): - """Return fragment IDs and the winning fragment's particle table row.""" - - name = "cluster_label" - - def __call__(self, trees: dict[str, Any]) -> ClusterLabelData: - """Build compact fragment supervision with instance and ancestor links. - - Parameters - ---------- - trees : dict[str, Any] - Raw event products returned by SuperaReader. - - Returns - ------- - ClusterLabelData - Compact fragment labels and a named particle-truth sidecar. - """ - # Reuse the same grid and overlap winners as the input tensor. - event = self.prepare(trees) - table = event["columns"] - num_particles = len(table["id"]) - host = table["frag_inst_id"] - valid = (host >= 0) & (host < num_particles) - shapes = _shape(table["sem_type"]) - shapes[valid] = _shape(table["inst_sem_type"][host[valid]]) - shapes[table["frag_sem_type"] == 5] = 4 - - # Keep source particle IDs while exposing event-local relationship fields. - particles = { - "particle": event["source_ids"].copy(), - "group": host.copy(), - "ancestor": table["ancestor_id"].copy(), - "interaction": table["interaction_id"].copy(), - "nu": np.full(num_particles, -1), - "pid": np.array( - [PDG_TO_PID.get(int(particle), -1) for particle in table["pdg"]] - ), - "group_primary": np.where(valid, table["id"] == host, -1), - "interaction_primary": (table["parent_id"] == table["id"]).astype(np.int64), - "vertex": ( - table["start_pos"][table["ancestor_id"], :3] / 10.0 - - event["meta"].lower - ) - / event["meta"].size, - "momentum": np.linalg.norm(table["start_mom"][:, :3], axis=1), - "energy_init": table["start_mom"][:, 3].copy(), - "shape": shapes, - } - - # The winning fragment row is also the compact particle association. - return ClusterLabelData( - coords=event["coords"].astype(self.itype), - features=np.column_stack( - (event["energy"], event["fragment"], event["fragment"]) - ).astype(self.ftype), - particles=particles, - meta=event["meta"], - remove_duplicates=False, - ) - - -class SuperaParticleParser(SuperaTensorParser): - """Translate the full stored particle table, preserving event-local IDs.""" - - name = "particle" - - def __call__(self, trees: dict[str, Any]) -> ObjectListData: - """Return SPINE particle truth in voxel units with MeV kinematics. - - Parameters - ---------- - trees : dict[str, Any] - Raw event products returned by SuperaReader. - - Returns - ------- - ObjectListData - Complete stored particle table in voxel units with MeV kinematics. - """ - # Reuse the same grid and overlap winners as the input tensor. - event = self.prepare(trees) - table, meta = event["columns"], event["meta"] - particles = [] - - # Deposited truth energy includes source contributions, not winner totals. - event_counts = np.bincount(event["instance"], minlength=len(table["id"])) - energy_prefix = np.r_[ - 0.0, - np.cumsum( - trees["points"][:, trees["metadata"]["columns"].index("dE")], - dtype=np.float64, - ), - ] - - # Preserve the full table so PPN and cluster references remain valid. - for row in range(len(table["id"])): - is_inst = table["inst_id"][row] == row - particle = Particle( - id=row, - parent_id=int(table["parent_id"][row]), - group_id=int(table["frag_inst_id"][row]), - interaction_id=int(table["interaction_id"][row]), - track_id=int(table["geant4_trackid"][row]), - pdg_code=int(table["pdg"][row]), - parent_pdg_code=int(table["parent_pdg"][row]), - shape=int( - _shape(table["inst_sem_type" if is_inst else "sem_type"][row]) - ), - energy_init=float(table["start_mom"][row, 3]), - momentum=table["start_mom"][row, :3].copy(), - units="px", - ) - - # Positions use voxel units; kinematic times remain in nanoseconds. - for source, target in ( - ("start", "position"), - ("end", "end_position"), - ("first", "first_step"), - ("last", "last_step"), - ): - setattr( - particle, - target, - ( - (table[source + "_pos"][row, :3] / 10.0 - meta.lower) - / meta.size - ).astype(self.ftype), - ) - particle.t, particle.end_t = float(table["start_pos"][row, 3]), float( - table["end_pos"][row, 3] - ) - particle.first_step_t, particle.last_step_t = float( - table["first_pos"][row, 3] - ), float(table["last_pos"][row, 3]) - particle.num_voxels = int(event_counts[row]) - if is_inst: - start, stop = table["inst_pc_start"][row], table["inst_pc_end"][row] - particle.energy_deposit = ( - float(energy_prefix[stop] - energy_prefix[start]) - if start >= 0 - else 0.0 - ) - particles.append(particle) - return ObjectListData(particles, Particle()) - - -class SuperaParticlePointsParser(SuperaTensorParser): - """PPN instance starts and track ends from collected-step kinematics.""" - - name = "particle_points" - - def __call__(self, trees: dict[str, Any]) -> TensorData: - """Omit unknown, LE, nonfinite and out-of-image endpoints. - - Parameters - ---------- - trees : dict[str, Any] - Raw event products returned by SuperaReader. - - Returns - ------- - TensorData - Instance starts and track ends with shape, particle and endpoint labels. - """ - # Reuse the same grid and overlap winners as the input tensor. - event = self.prepare(trees) - table, meta = event["columns"], event["meta"] - rows = [] - - # Only instances represented in the voxel image can provide targets. - for row in np.unique(event["instance"]): - shape = int(_shape(table["inst_sem_type"][row])) - if shape not in (0, 1, 2, 3) or abs(table["pdg"][row]) > 1000000000: - continue - for endpoint, key in enumerate( - ("first_pos", "last_pos") if shape == 1 else ("first_pos",) - ): - point = (table[key][row, :3] / 10.0 - meta.lower) / meta.size - if ( - np.isfinite(point).all() - and np.all(point >= 0) - and np.all(point < meta.count) - ): - rows.append([*point, shape, row, endpoint]) - - # Preserve a stable six-column layout even when no endpoints survive. - data = np.asarray(rows, dtype=self.ftype).reshape(-1, 6) - return TensorData( - coords=data[:, :3], - features=data[:, 3:], - meta=meta, - coordinate_groups={"point": (0, 1, 2)}, - feature_fields={"shape": (0,), "particle": (1,), "endpoint": (2,)}, - index_cols=np.array([1]), - index_shifts=np.array([len(table["id"])]), - ) diff --git a/src/spine/io/parse/supera/cluster.py b/src/spine/io/parse/supera/cluster.py new file mode 100644 index 00000000..f4420472 --- /dev/null +++ b/src/spine/io/parse/supera/cluster.py @@ -0,0 +1,45 @@ +"""Compact fragmentation labels with event-local particle relationships.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.data import ClusterLabelData + +from .base import SuperaParserBase + +__all__ = ["SuperaClusterLabelParser"] + + +class SuperaClusterLabelParser(SuperaParserBase): + """Return fragment IDs and the winning fragment's particle table row.""" + + name = "cluster_label" + + def __call__(self, trees: dict[str, Any]) -> ClusterLabelData: + """Build compact fragment supervision with instance and ancestor links. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + ClusterLabelData + Compact fragment labels and a named particle-truth sidecar. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + # The winning fragment row is also the compact particle association. + return ClusterLabelData( + coords=event["coords"].astype(self.itype), + features=np.column_stack( + (event["energy"], event["fragment"], event["fragment"]) + ).astype(self.ftype), + particles=event["labels"], + meta=event["meta"], + remove_duplicates=False, + ) diff --git a/src/spine/io/parse/supera/metadata.py b/src/spine/io/parse/supera/metadata.py new file mode 100644 index 00000000..1cad058d --- /dev/null +++ b/src/spine/io/parse/supera/metadata.py @@ -0,0 +1,96 @@ +"""Shared image metadata and native event/particle provenance.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.data import Meta, TensorData + +from ..base import ParserBase +from .base import SuperaParserBase + +__all__ = ["SuperaMetaParser", "SuperaSourceIDsParser", "SuperaEventInfoParser"] + + +class SuperaMetaParser(SuperaParserBase): + """Expose the shared image transform for truth construction in cm.""" + + name = "meta" + + def __call__(self, trees: dict[str, Any]) -> Meta: + """Return the metadata shared by all native spatial products. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + Meta + Detector bounds and voxel dimensions in centimetres. + """ + return self.prepare(trees)["meta"] + + +class SuperaSourceIDsParser(SuperaParserBase): + """Preserve sparse source IDs separately from internal particle indexes.""" + + name = "source_ids" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return one source identifier per particle-table row. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + TensorData + Integer provenance IDs; batching never shifts these identifiers. + """ + return TensorData( + features=self.prepare(trees)["source_ids"][:, None], + feature_fields={"source_id": (0,)}, + ) + + +class SuperaEventInfoParser(ParserBase): + """Preserve native event/job identities without fabricating run/subrun IDs.""" + + name = "event_info" + + def __init__(self, dtype: str) -> None: + """Request the small event-identity product. + + Parameters + ---------- + dtype : str + Dataset floating-point dtype; identities retain integer precision. + """ + super().__init__(dtype, identity_event="events") + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return all stored identity fields as a named integer tensor. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + TensorData + One row containing the file's event, input and job identities. + """ + fields = sorted(trees["events"]) + return TensorData( + features=np.array( + [[trees["events"][key] for key in fields]], dtype=np.int64 + ), + feature_fields={key: (column,) for column, key in enumerate(fields)}, + ) diff --git a/src/spine/io/parse/supera/particle.py b/src/spine/io/parse/supera/particle.py new file mode 100644 index 00000000..2f567989 --- /dev/null +++ b/src/spine/io/parse/supera/particle.py @@ -0,0 +1,282 @@ +"""Particle truth, coordinate labels, genealogy and PPN targets.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.data import EdgeIndexData, Meta, ObjectListData, Particle, TensorData + +from .base import SuperaParserBase, _shape + +__all__ = [ + "SuperaParticleParser", + "SuperaParticlePointsParser", + "SuperaParticleCoordsParser", + "SuperaParticleGraphParser", +] + + +class SuperaParticleParser(SuperaParserBase): + """Translate the full stored particle table, preserving event-local IDs.""" + + name = "particle" + + def __call__(self, trees: dict[str, Any]) -> ObjectListData: + """Return SPINE particle truth in voxel units with MeV kinematics. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + ObjectListData + Complete stored particle table in voxel units with MeV kinematics. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table = event["columns"] + particles = [] + + # Charge is unavailable here. Integrate truth dE before overlap merging. + event_counts = np.bincount(event["fragment"], minlength=len(table["id"])) + energy_prefix = np.r_[ + 0.0, + np.cumsum( + trees["points"][:, trees["metadata"]["columns"].index("dE")], + dtype=np.float64, + ), + ] + + starts, ends = self.endpoints(trees) + + # Preserve the full table so PPN and cluster references remain valid. + for row in range(len(table["id"])): + particle = Particle( + id=row, + parent_id=int(table["parent_id"][row]), + group_id=int(table["frag_inst_id"][row]), + interaction_id=int(event["labels"]["interaction"][row]), + track_id=int(table["geant4_trackid"][row]), + pdg_code=int(table["pdg"][row]), + parent_pdg_code=int(table["parent_pdg"][row]), + shape=int(event["labels"]["shape"][row]), + energy_init=float(table["start_mom"][row, 3]), + momentum=table["start_mom"][row, :3].copy(), + pid=int(event["labels"]["pid"][row]), + group_primary=int(event["labels"]["group_primary"][row]), + interaction_primary=int(event["labels"]["interaction_primary"][row]), + ancestor_position=event["labels"]["vertex"][row].astype(self.ftype), + children_id=np.flatnonzero( + (table["parent_id"] == row) & (table["id"] != row) + ).astype(np.int32), + end_momentum=table["end_mom"][row, :3].copy(), + units="px", + ) + + # Positions use voxel units; kinematic times remain in nanoseconds. + for positions, target in ( + (table["start_pos"], "position"), + (table["end_pos"], "end_position"), + (starts, "first_step"), + (ends, "last_step"), + ): + setattr( + particle, + target, + ( + (positions[row, :3] / 10.0 - event["meta"].lower) + / event["meta"].size + ).astype(self.ftype), + ) + particle.t, particle.end_t = float(table["start_pos"][row, 3]), float( + table["end_pos"][row, 3] + ) + particle.first_step_t, particle.last_step_t = float(starts[row, 3]), float( + ends[row, 3] + ) + self._fill_ancestry(particle, row, table, event["meta"]) + particle.num_voxels = int(event_counts[row]) + # Own-particle ranges need not cover deposits absorbed from omitted + # source particles. Fragment ranges do cover all deposits; assign + # each fragment's energy once, to its stored representative. + start, stop = table["frag_pc_start"][row], table["frag_pc_end"][row] + particle.energy_deposit = 0.0 + if table["frag_id"][row] == row and start >= 0: + particle.energy_deposit = float( + energy_prefix[stop] - energy_prefix[start] + ) + particles.append(particle) + return ObjectListData(particles, Particle()) + + def _fill_ancestry( + self, particle: Particle, row: int, table: dict[str, np.ndarray], meta: Meta + ) -> None: + """Populate available genealogy without dereferencing missing links. + + Parameters + ---------- + particle : Particle + Output truth particle, modified in place. + row : int + Event-local table row. + table : dict[str, np.ndarray] + Particle columns with normalized relationships. + meta : Meta + Shared detector transform in centimetres. + """ + # Missing links must never index the last particle through -1. + for relation in ("parent", "ancestor"): + relative = int(table[relation + "_id"][row]) + if relative >= 0: + setattr( + particle, + relation + "_track_id", + int(table["geant4_trackid"][relative]), + ) + setattr(particle, relation + "_pdg_code", int(table["pdg"][relative])) + setattr( + particle, + relation + "_t", + float(table["start_pos"][relative, 3]), + ) + parent = int(table["parent_id"][row]) + if parent >= 0: + particle.parent_position = ( + (table["start_pos"][parent, :3] / 10.0 - meta.lower) / meta.size + ).astype(self.ftype) + if "geant4_parent_trackid" in table: + particle.parent_track_id = int(table["geant4_parent_trackid"][row]) + particle.parent_pdg_code = int(table["geant4_parent_pdg"][row]) + + +class SuperaParticlePointsParser(SuperaParserBase): + """PPN instance starts and track ends from collected-step kinematics.""" + + name = "particle_points" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Omit unknown, LE, nonfinite and out-of-image endpoints. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + TensorData + Instance starts and track ends with shape, particle and endpoint labels. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table, meta = event["columns"], event["meta"] + rows = [] + + # Only instances represented in the voxel image can provide targets. + for row in np.unique(event["instance"]): + shape = int(_shape(table["inst_sem_type"][row])) + if shape not in (0, 1, 2, 3) or abs(table["pdg"][row]) > 1000000000: + continue + for endpoint, key in enumerate( + ("first_pos", "last_pos") if shape == 1 else ("first_pos",) + ): + point = (table[key][row, :3] / 10.0 - meta.lower) / meta.size + if ( + np.isfinite(point).all() + and np.all(point >= 0) + and np.all(point < meta.count) + ): + rows.append([*point, shape, row, endpoint]) + + # Preserve a stable six-column layout even when no endpoints survive. + data = np.asarray(rows, dtype=self.ftype).reshape(-1, 6) + return TensorData( + coords=data[:, :3], + features=data[:, 3:], + meta=meta, + coordinate_groups={"point": (0, 1, 2)}, + feature_fields={"shape": (0,), "particle": (1,), "endpoint": (2,)}, + index_cols=np.array([1]), + index_shifts=np.array([len(table["id"])]), + ) + + +class SuperaParticleCoordsParser(SuperaParserBase): + """One pair of collected-step endpoints per stored particle-table row. + + Instance representatives use upstream collected-step kinematics. Other + rows use their fragment's earliest/latest deposits. Missing instance + endpoints use its non-LE deposits (all deposits if none are non-LE). + These fallbacks are voxelized deposit positions, not exact step boundaries. + Non-track ends equal their starts. Unavailable coordinates and + times remain NaN; no creation/destruction positions are substituted. + """ + + name = "particle_coords" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return aligned start/end coordinates and time/shape features. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + TensorData + Six coordinate columns and two features, with one row per particle. + Point times are converted from microseconds to nanoseconds. + """ + event = self.prepare(trees) + meta = event["meta"] + starts, ends = self.endpoints(trees) + shapes = event["labels"]["shape"] + coords = np.column_stack( + ( + (starts[:, :3] / 10.0 - meta.lower) / meta.size, + (ends[:, :3] / 10.0 - meta.lower) / meta.size, + ) + ) + coords[~np.isfinite(coords)] = np.nan + return TensorData( + coords=coords.astype(self.ftype), + features=np.column_stack((starts[:, 3], shapes)).astype(self.ftype), + meta=meta, + coordinate_groups={"start": (0, 1, 2), "end": (3, 4, 5)}, + feature_fields={"time": (0,), "shape": (1,)}, + ) + + +class SuperaParticleGraphParser(SuperaParserBase): + """Directed stored-parent links in the event-local particle namespace. + + Truth construction contracts invisible rows and groups using this graph. + Retain every stored link so contraction can recover visible descendants. + """ + + name = "particle_graph" + + def __call__(self, trees: dict[str, Any]) -> EdgeIndexData: + """Return parent-to-child edges, excluding roots and missing parents. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + EdgeIndexData + Directed edges with a span equal to the full particle-table length. + """ + table = self.prepare(trees)["columns"] + parents, children = table["parent_id"], table["id"] + valid = (parents >= 0) & (parents != children) + return EdgeIndexData( + features=np.stack((parents[valid], children[valid])), span=len(children) + ) diff --git a/src/spine/io/parse/supera/tensor.py b/src/spine/io/parse/supera/tensor.py new file mode 100644 index 00000000..7e83ba23 --- /dev/null +++ b/src/spine/io/parse/supera/tensor.py @@ -0,0 +1,65 @@ +"""Aligned input tensors and semantic segmentation labels.""" + +from __future__ import annotations + +from typing import Any + +from spine.data import TensorData + +from .base import SuperaParserBase + +__all__ = ["SuperaTensorParser", "SuperaSegmentationParser"] + + +class SuperaTensorParser(SuperaParserBase): + """Return voxelized truth-deposit energy in MeV.""" + + name = "tensor" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return summed truth energy in MeV on the shared sparse grid. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + TensorData + Sparse coordinates and summed deposit energy in MeV. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + return TensorData( + coords=event["coords"].astype(self.itype), + features=event["energy"].astype(self.ftype)[:, None], + meta=event["meta"], + ) + + +class SuperaSegmentationParser(SuperaParserBase): + """Return semantic labels in exactly the input voxel order.""" + + name = "segmentation" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return five-class labels; unknown point semantics are rejected. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by SuperaReader. + + Returns + ------- + TensorData + Semantic labels sharing the input tensor coordinates and row order. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + return TensorData( + coords=event["coords"].astype(self.itype), + features=event["shape"].astype(self.ftype)[:, None], + meta=event["meta"], + ) diff --git a/test/test_io/test_supera/__init__.py b/test/test_io/test_supera/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_io/test_supera/conftest.py b/test/test_io/test_supera/conftest.py new file mode 100644 index 00000000..2c8ad8d3 --- /dev/null +++ b/test/test_io/test_supera/conftest.py @@ -0,0 +1,152 @@ +"""Shared native Supera fixtures and complete schema.""" + +import h5py +import numpy as np +import pytest + +FULL_SCHEMA = { + key: {"provider": provider} + for key, provider in ( + ("data", "tensor"), + ("seg_label", "segmentation"), + ("clust_label", "cluster_label"), + ("ppn_label", "particle_points"), + ("particles", "particle"), + ("coord_label", "particle_coords"), + ("meta", "meta"), + ("graph_label", "particle_graph"), + ("source_ids", "source_ids"), + ("event_info", "event_info"), + ) +} + + +@pytest.fixture +def sample(tmp_path): + """Two events with sparse IDs, overlap, LE in a track and empty truth.""" + path = tmp_path / "supera.h5" + ids = np.array([10, 80, 99, 120, 150], dtype=np.int32) + # Track, shower, LE fragment of track, delta, Michel. + table = dict( + id=ids, + parent_id=ids, + ancestor_id=ids, + frag_id=ids, + inst_id=np.array([10, 80, -1, 120, 150]), + frag_inst_id=np.array([10, 80, 10, 120, 150]), + sem_type=np.array([2, 1, 5, 3, 4]), + frag_sem_type=np.array([2, 1, 5, 3, 4]), + inst_sem_type=np.array([2, 1, -1, 3, 4]), + pc_start=np.array([0, 3, 2, 4, 5]), + pc_end=np.array([2, 4, 3, 5, 6]), + frag_pc_start=np.array([0, 3, 2, 4, 5]), + frag_pc_end=np.array([2, 4, 3, 5, 6]), + inst_pc_start=np.array([0, 3, -1, 4, 5]), + inst_pc_le_start=np.array([2, 4, -1, 5, 6]), + inst_pc_end=np.array([3, 4, -1, 5, 6]), + pdg=np.array([13, 11, 11, 11, 11]), + parent_pdg=np.array([13] * 5), + geant4_trackid=ids, + interaction_id=np.zeros(5, dtype=int), + mass=np.ones(5), + ) + for key in ( + "start_pos", + "end_pos", + "first_pos", + "last_pos", + "start_mom", + "end_mom", + ): + table[key] = np.tile([1.5, 1.5, 1.5, 10.0], (5, 1)) + table["last_pos"][0, :3] = [4.5, 1.5, 1.5] + table["first_pos"][3, :3] = [10.5, 1.5, 1.5] + table["first_pos"][4, :3] = [13.5, 1.5, 1.5] + table["last_pos"][1] = np.nan + points = np.array( + [ + [1.5, 1.5, 1.5, 5, 2, 0], + [4.5, 1.5, 1.5, 2, 3, 0], + [7.5, 1.5, 1.5, 1, 4, 0], + [1.5, 1.5, 1.5, 0, 9, 0], + [10.5, 1.5, 1.5, 1, 5, 0], + [13.5, 1.5, 1.5, 1, 6, 0], + ], + dtype=np.float32, + ) + with h5py.File(path, "w") as file: + file["format_version"] = "4.1.0" + file["n_events"] = 3 + file.attrs["voxel_size"] = "3.0" + file.attrs["hit_energy"] = "charge" + file["events/offsets"] = [0, 5, 10, 10] + file["events/event_id"] = [11, 12, 13] + file["points/offsets"] = [0, 6, 12, 12] + file["points/flat"] = np.concatenate([points, points]) + file["points/columns"] = np.array( + ["x", "y", "z", "t", "dE", "dX"], dtype=h5py.string_dtype() + ) + for key, val in table.items(): + second = np.where(val >= 0, val + 6, val) if "pc_" in key else val + file["particles/" + key] = np.concatenate([val, second]) + file["inter/offsets"] = [0, 1, 2, 2] + file["inter/interaction_id"] = [0, 0] + file["inter/part_start"] = [0, 5] + file["inter/part_end"] = [5, 10] + file["inter/pc_start"] = [0, 6] + file["inter/pc_end"] = [6, 12] + file["groups/offsets"] = [0, 9, 18, 18] + file["groups/fragment_id"] = [10] * 18 + file["groups/is_le"] = [False] * 18 + file["groups/volume_offsets_index"] = [0, 2, 4, 6] + file["groups/volume_offsets"] = [0, 4, 0, 4, 0, 0] + # A huge virtual dataset: a projected read must never access it. + hit_plane = file.create_group("hit_labels/volume0/plane0") + hit_plane["offsets"] = [0, 2, 4, 4] + hit_plane.create_dataset( + "fragment_id", shape=(10**9,), dtype="i4", fillvalue=10 + ) + return str(path) + + +def round_trip_cache(batch, products, path, stage): + """Publish native products and reload them through cache collation. + + Parameters + ---------- + batch : dict + Collated input batch containing physical-source provenance. + products : dict + Batched products to cache, without source metadata. + path : pathlib.Path + Temporary cache repository. + stage : str + Cache stage name. + + Returns + ------- + dict + Re-collated products from the published cache stage. + """ + from spine.io.collate import CollateAll + from spine.io.read import CacheReader + from spine.io.unwrap import Unwrapper + from spine.io.write import CacheWriter + + metadata = { + key: value + for key, value in batch.items() + if key == "index" or key.startswith("source_file_") + } + payload = { + key: value.to_numpy() if hasattr(value, "to_numpy") else value + for key, value in products.items() + } + writer = CacheWriter(path=str(path), stage=stage, keys=list(products)) + writer(Unwrapper()({**metadata, **payload}), {}) + writer.finalize() + writer.close() + reader = CacheReader(path=str(path), stage=stage) + events = reader.get_many(list(range(len(batch["index"])))) + reader.close() + return CollateAll(data_keys=products)(events) diff --git a/test/test_io/test_supera/test_supera.py b/test/test_io/test_supera/test_supera.py index 87ec951f..c59eaf7e 100644 --- a/test/test_io/test_supera/test_supera.py +++ b/test/test_io/test_supera/test_supera.py @@ -24,92 +24,6 @@ } -@pytest.fixture -def sample(tmp_path): - """Two events with sparse IDs, overlap, LE in a track and empty truth.""" - path = tmp_path / "supera.h5" - ids = np.array([10, 80, 99, 120, 150], dtype=np.int32) - # Track, shower, LE fragment of track, delta, Michel. - table = dict( - id=ids, - parent_id=ids, - ancestor_id=ids, - frag_id=ids, - inst_id=np.array([10, 80, -1, 120, 150]), - frag_inst_id=np.array([10, 80, 10, 120, 150]), - sem_type=np.array([2, 1, 5, 3, 4]), - frag_sem_type=np.array([2, 1, 5, 3, 4]), - inst_sem_type=np.array([2, 1, -1, 3, 4]), - frag_pc_start=np.array([0, 3, 2, 4, 5]), - frag_pc_end=np.array([2, 4, 3, 5, 6]), - inst_pc_start=np.array([0, 3, -1, 4, 5]), - inst_pc_le_start=np.array([2, 4, -1, 5, 6]), - inst_pc_end=np.array([3, 4, -1, 5, 6]), - pdg=np.array([13, 11, 11, 11, 11]), - parent_pdg=np.array([13] * 5), - geant4_trackid=ids, - interaction_id=np.zeros(5, dtype=int), - mass=np.ones(5), - ) - for key in ( - "start_pos", - "end_pos", - "first_pos", - "last_pos", - "start_mom", - "end_mom", - ): - table[key] = np.tile([1.5, 1.5, 1.5, 10.0], (5, 1)) - table["last_pos"][0, :3] = [4.5, 1.5, 1.5] - table["first_pos"][3, :3] = [10.5, 1.5, 1.5] - table["first_pos"][4, :3] = [13.5, 1.5, 1.5] - table["last_pos"][1] = np.nan - points = np.array( - [ - [1.5, 1.5, 1.5, 5, 2, 0], - [4.5, 1.5, 1.5, 2, 3, 0], - [7.5, 1.5, 1.5, 1, 4, 0], - [1.5, 1.5, 1.5, 0, 9, 0], - [10.5, 1.5, 1.5, 1, 5, 0], - [13.5, 1.5, 1.5, 1, 6, 0], - ], - dtype=np.float32, - ) - with h5py.File(path, "w") as file: - file["format_version"] = "4.1.0" - file["n_events"] = 3 - file.attrs["voxel_size"] = "3.0" - file.attrs["hit_energy"] = "charge" - file["events/offsets"] = [0, 5, 10, 10] - file["events/event_id"] = [11, 12, 13] - file["points/offsets"] = [0, 6, 12, 12] - file["points/flat"] = np.concatenate([points, points]) - file["points/columns"] = np.array( - ["x", "y", "z", "t", "dE", "dX"], dtype=h5py.string_dtype() - ) - for key, val in table.items(): - second = np.where(val >= 0, val + 6, val) if "pc_" in key else val - file["particles/" + key] = np.concatenate([val, second]) - file["inter/offsets"] = [0, 1, 2, 2] - file["inter/interaction_id"] = [0, 0] - file["inter/part_start"] = [0, 5] - file["inter/part_end"] = [5, 10] - file["inter/pc_start"] = [0, 6] - file["inter/pc_end"] = [6, 12] - file["groups/offsets"] = [0, 9, 18, 18] - file["groups/fragment_id"] = [10] * 18 - file["groups/is_le"] = [False] * 18 - file["groups/volume_offsets_index"] = [0, 2, 4, 6] - file["groups/volume_offsets"] = [0, 4, 0, 4, 0, 0] - # A huge virtual dataset: a projected read must never access it. - hit_plane = file.create_group("hit_labels/volume0/plane0") - hit_plane["offsets"] = [0, 2, 4, 4] - hit_plane.create_dataset( - "fragment_id", shape=(10**9,), dtype="i4", fillvalue=10 - ) - return str(path) - - def dataset(path, **kwargs): """Build the complete parser schema for one synthetic input file.""" return SuperaDataset(file_keys=path, dtype="float32", schema=SCHEMA, **kwargs) @@ -246,8 +160,9 @@ def test_grid_transform_and_particle_energy(sample): np.testing.assert_allclose(event["ppn_label"].coordinate_data[0], [0.5, 0.5, 0.5]) np.testing.assert_allclose(event["data"].meta.lower, [0, 0, 0]) # The winning track sees 11 MeV in the overlapped input cell, but its - # source-instance truth includes only its own 2 + 3 + 4 MeV contributions. - assert event["particles"][0].energy_deposit == 9 + # source-fragment truth includes only its own 2 + 3 MeV contributions. + # The separate LE fragment contributes 4 MeV when building the truth group. + assert event["particles"][0].energy_deposit == 5 def test_registered_factories(sample): @@ -322,6 +237,8 @@ def test_empty_representative_retains_particle_row(sample): raw = SuperaReader(file_keys=sample)[0] raw["points"] = raw["points"][:5] for key in ( + "pc_start", + "pc_end", "frag_pc_start", "frag_pc_end", "inst_pc_start", diff --git a/test/test_io/test_supera/test_supera_chain.py b/test/test_io/test_supera/test_supera_chain.py new file mode 100644 index 00000000..5c3cbd36 --- /dev/null +++ b/test/test_io/test_supera/test_supera_chain.py @@ -0,0 +1,156 @@ +"""Opt-in real-sample downstream training and truth conversion integration.""" + +import os +from copy import deepcopy +from pathlib import Path + +import numpy as np +import pytest + +from spine.io.collate import CollateAll +from spine.io.dataset import SuperaDataset + +from .conftest import FULL_SCHEMA, round_trip_cache +from .test_supera_truth import check_truth_round_trip + +SAMPLE = os.environ.get("SPINE_SUPERA_SAMPLE") +pytestmark = pytest.mark.skipif(not SAMPLE, reason="Set SPINE_SUPERA_SAMPLE") + + +def test_real_truth_conversion(tmp_path): + """Build two complete real events and preserve cm geometry and associations.""" + ds = SuperaDataset(file_keys=SAMPLE, schema=FULL_SCHEMA, dtype="float32", n_entry=2) + events = [ds[0], ds[1]] + truth = check_truth_round_trip(events, tmp_path / "truth.h5") + for index, event in enumerate(events): + raw = ds.reader[index] + interactions = raw["inter"] + for interaction in truth["truth_interactions"][index]: + row = np.flatnonzero(interactions["interaction_id"] == interaction.orig_id)[ + 0 + ] + expected = ( + np.array([interactions[key][row] for key in ("x", "y", "z")]) / 10 + ) + np.testing.assert_allclose(interaction.vertex, expected, atol=1e-4) + # Fragment-attributed energy counts every source deposit exactly once. + energy_col = raw["metadata"]["columns"].index("dE") + assert np.isclose( + sum(p.energy_deposit for p in event["particles"]), + raw["points"][:, energy_col].sum(), + rtol=1e-5, + ) + assert len(truth["truth_fragments"][index]) > 0 + assert len(truth["truth_particles"][index]) > 0 + assert any(len(p.children_id) for p in truth["truth_particles"][index]) + + +@pytest.mark.parametrize("path", ["shower", "track", "inter"]) +def test_real_grappa_targets_and_backward(path, tmp_path): + """Every generic GrapPA objective has eligible targets and nonzero gradients.""" + torch = pytest.importorskip("torch") + pytest.importorskip("torch_geometric") + from spine.config import load_config_file + from spine.data import TensorBatch + from spine.model.grappa import GrapPA, GrapPALoss + + torch.manual_seed(7) + ds = SuperaDataset(file_keys=SAMPLE, schema=FULL_SCHEMA, dtype="float32", n_entry=2) + # Two events exercise the coordinate-table offsets used by orientation targets. + batch = CollateAll(data_keys=ds.data_keys)([ds[0], ds[1]]) + labels = batch["clust_label"].to_tensor(dtype=torch.float32) + coords = batch["coord_label"].to_tensor(dtype=torch.float32) + config_path = ( + Path(__file__).resolve().parents[3] + / "config/full_chain/full_chain_regression.yaml" + ) + config = load_config_file(str(config_path), download=False)["model"]["modules"] + model_config = deepcopy(config["grappa_" + path]) + loss_config = deepcopy(config["grappa_" + path + "_loss"]) + # Keep every objective and geometry feature; shrink only the neural hidden width. + gnn = model_config["gnn_model"] + gnn["edge_layer"]["mlp"]["width"] = 8 + for name in ("message_mlp", "aggr_mlp"): + gnn["node_layer"][name]["width"] = 8 + if path == "inter": + loss_config["node_loss"]["primary"]["use_closest"] = True + loss_config["return_targets"] = True + model = GrapPA(model_config).train() + objective = GrapPALoss(loss_config, model_config) + + graph = model.materialize_graph(data=labels, coord_label=coords) + targets = objective.materialize_targets( + clust_label=labels, coord_label=coords, **graph + ) + for name, valid in targets.items(): + if name.endswith("_valid"): + mask = valid.numpy_tensor().astype(bool) + assert mask.sum() > 0, (path, name) + values = targets[name.removesuffix("_valid") + "_target"].numpy_tensor() + assert np.all(values[mask] >= 0), (path, name) + + # Publish fragment/particle graph targets with their real source identities. + graph_keys = ( + "clusts", + "edge_index", + "node_features", + "edge_features", + "shapes", + "start_points", + "end_points", + ) + cached = round_trip_cache( + batch, + {**{key: graph[key] for key in graph_keys if key in graph}, **targets}, + tmp_path / "chain.spine-cache", + "particle_graph" if path == "inter" else "fragment_graph", + ) + cached = { + key: ( + value.to_tensor(dtype=torch.float32) + if isinstance(value, TensorBatch) + else value + ) + for key, value in cached.items() + } + for name in targets: + np.testing.assert_array_equal( + cached[name].to_numpy().numpy_tensor(), targets[name].numpy_tensor() + ) + + # Train from the graph cache without replacing or dropping any objective. + output = model( + **{ + key: cached[key] + for key in ( + "clusts", + "edge_index", + "node_features", + "edge_features", + "shapes", + ) + if key in cached + } + ) + output = {**cached, **output} + metrics = objective(clust_label=labels, coord_label=coords, **output) + report = {} + for name, loss in metrics.items(): + if name.endswith("_loss"): + prefix = name.removesuffix("_loss") + assert metrics[prefix + "_count"] > 0, (path, prefix) + assert torch.isfinite(loss), (path, prefix) + gradients = torch.autograd.grad( + loss, tuple(model.parameters()), retain_graph=True, allow_unused=True + ) + assert all(torch.isfinite(g).all() for g in gradients if g is not None) + assert any(g is not None and g.abs().sum() > 0 for g in gradients), ( + path, + prefix, + ) + report[prefix] = { + "count": metrics[prefix + "_count"], + "loss": float(loss.detach()), + } + metrics["loss"].backward() + print({"path": path, "objectives": report}) diff --git a/test/test_io/test_supera/test_supera_model.py b/test/test_io/test_supera/test_supera_model.py index 36581b47..23aa083e 100644 --- a/test/test_io/test_supera/test_supera_model.py +++ b/test/test_io/test_supera/test_supera_model.py @@ -8,19 +8,23 @@ import numpy as np import pytest +from .conftest import round_trip_cache + SAMPLE = os.environ.get("SPINE_SUPERA_SAMPLE") pytestmark = pytest.mark.skipif( not SAMPLE, reason="Set SPINE_SUPERA_SAMPLE for real-file training" ) -def test_real_supera_forward_backward(): +def test_real_supera_forward_backward(tmp_path): """Backpropagate every training objective through the complete real event.""" torch = pytest.importorskip("torch") pytest.importorskip("MinkowskiEngine") pytest.importorskip("torch_cluster") from spine.io.collate import CollateAll from spine.io.dataset import SuperaDataset + from spine.model.full_chain.providers.segmentation import SegmentationStage + from spine.model.full_chain.state import ChainState from spine.model.graph_spice import GraphSPICE, GraphSPICELoss from spine.model.uresnet.ppn import UResNetPPN, UResNetPPNLoss @@ -44,6 +48,7 @@ def test_real_supera_forward_backward(): e["seg_label"].features[:, 0], e["clust_label"].shapes ) # Keep the full real event, including all overlaps and endpoint targets. + source_batch = CollateAll(data_keys=ds.data_keys)([e]) batch = CollateAll(data_keys=schema)([e]) batch = {k: v.to_tensor(dtype=torch.float32) for k, v in batch.items()} size = int(e["data"].meta.count.max()) + 2 @@ -60,8 +65,28 @@ def test_real_supera_forward_backward(): # Segmentation and PPN share the standard UResNet backbone on main. model = UResNetPPN(backbone, ppn={}).train() - objective = UResNetPPNLoss(backbone, uresnet_loss={}, ppn={}, ppn_loss={}) - metrics = objective(batch["seg_label"], batch["ppn_label"], **model(batch["data"])) + objective = UResNetPPNLoss( + backbone, uresnet_loss={}, ppn={}, ppn_loss={"return_mask_labels": True} + ) + stage = SegmentationStage("segmentation", "uresnet", model, None) + output = stage(ChainState(**batch)).outputs + metrics = objective(batch["seg_label"], batch["ppn_label"], **output) + cached = round_trip_cache( + source_batch, + {"seg_pred": output["seg_pred"], "ppn_points": output["ppn_points"]}, + tmp_path / "segmentation.spine-cache", + "segmentation", + ) + for name in cached: + np.testing.assert_allclose( + cached[name].to_numpy().numpy_tensor(), + output[name].to_numpy().numpy_tensor(), + ) + assert len(e["seg_label"].features) > 0 + assert len(e["ppn_label"].features) > 0 + positive_points = int(metrics["ppn_mask_labels"][0].torch_tensor().sum()) + assert positive_points > 0 + assert metrics["ppn_type_loss"] > 0 and metrics["ppn_reg_loss"] > 0 parameter = next(model.uresnet.backbone.parameters()) report = {} for name in ("uresnet_loss", "ppn_loss"): @@ -93,6 +118,7 @@ def test_real_supera_forward_backward(): batch["clust_label"], **graph(batch["data"], batch["seg_label"]), ) + assert graph_metrics["count"] > 0 assert torch.isfinite(graph_metrics["loss"]) graph_parameter = next(graph.embedder.backbone.parameters()) grad = torch.autograd.grad( @@ -111,6 +137,8 @@ def test_real_supera_forward_backward(): { "voxels": len(e["data"].features), "ppn_targets": len(e["ppn_label"].features), + "ppn_positive_voxels": positive_points, + "graph_spice_edges": graph_metrics["count"], "losses": report, "total_loss": float(total.detach()), } diff --git a/test/test_io/test_supera/test_supera_truth.py b/test/test_io/test_supera/test_supera_truth.py new file mode 100644 index 00000000..6de56909 --- /dev/null +++ b/test/test_io/test_supera/test_supera_truth.py @@ -0,0 +1,331 @@ +"""Downstream truth contracts, provenance and physical-coordinate round trips.""" + +import copy + +import numpy as np + +from spine.construct import BuildManager +from spine.io.collate import CollateAll +from spine.io.dataset import SuperaDataset +from spine.io.parse.supera import ( + SuperaClusterLabelParser, + SuperaParticleCoordsParser, + SuperaParticleGraphParser, + SuperaParticleParser, +) +from spine.io.read import HDF5Reader, SuperaReader +from spine.io.unwrap import Unwrapper +from spine.io.write import HDF5Writer + +from .conftest import FULL_SCHEMA + + +def test_local_ids_coordinates_and_provenance(sample): + """Sparse source IDs cannot leak into row-indexed supervision, even in batches.""" + ds = SuperaDataset(file_keys=sample, schema=FULL_SCHEMA, dtype="float32") + events = [ds[i] for i in (0, 2, 1)] + batch = CollateAll(data_keys=ds.data_keys)(events) + unwrapped = Unwrapper()(batch) + for i, event in enumerate(events): + count = len(event["particles"]) + np.testing.assert_array_equal( + unwrapped["clust_label"][i].particle_field("particle"), np.arange(count) + ) + assert unwrapped["coord_label"][i].coordinate_data.shape == (count, 6) + np.testing.assert_allclose(unwrapped["meta"][i].size, 0.3) + np.testing.assert_array_equal( + unwrapped["source_ids"][i].features, event["source_ids"].features + ) + assert unwrapped["event_info"][i].features[0, 0] == [11, 13, 12][i] + if count: + coords = event["coord_label"].coordinate_data + np.testing.assert_allclose(coords[0], [0.5, 0.5, 0.5, 1.5, 0.5, 0.5]) + np.testing.assert_allclose(coords[1, :3], coords[1, 3:]) + assert event["source_ids"].features.max() > count + for row, particle in enumerate(event["particles"]): + labels = event["clust_label"].particles + assert particle.pid == labels["pid"][row] + assert particle.group_primary == labels["group_primary"][row] + assert ( + particle.interaction_primary == labels["interaction_primary"][row] + ) + + +def test_missing_endpoints_and_ancestry(sample): + """Missing links cannot wrap to the last row, and deposited endpoints are usable.""" + raw = SuperaReader(file_keys=sample)[0] + table = raw["particles"] + table["parent_id"][0] = -1 + table["ancestor_id"][0] = -1 + table["first_pos"][:2] = np.nan + table["last_pos"][0] = np.inf + table["first_pos"][2] = np.nan + table["last_pos"][2] = np.nan + # An unobserved extra row has neither endpoints nor a recoverable fragment. + for key, values in table.items(): + table[key] = np.concatenate((values, values[-1:])) + table["id"][-1] = 100000 + for key in ("parent_id", "ancestor_id", "frag_id", "inst_id", "frag_inst_id"): + table[key][-1] = -1 + for key in ("first_pos", "last_pos"): + table[key][-1] = np.nan + table["pc_start"][-1] = table["pc_end"][-1] = -1 + coords = SuperaParticleCoordsParser("float32")(raw) + assert np.isfinite(coords.coordinate_data[:5]).all() + assert np.isnan(coords.coordinate_data[-1]).all() + # Raw point times are microseconds; coordinate-label times are nanoseconds. + assert coords.features[0, 0] == 2000 + np.testing.assert_allclose(coords.coordinate_data[0, :3], [1.5, 0.5, 0.5]) + labels = SuperaClusterLabelParser("float32")(raw).particles + assert labels["interaction_primary"][0] == -1 + assert np.isnan(labels["vertex"][0]).all() + particles = SuperaParticleParser("float32")(raw) + assert particles[0].ancestor_track_id == -1 + assert np.isnan(particles[0].parent_position).all() + assert particles[-1].energy_deposit == 0 + assert SuperaParticleGraphParser("float32")(raw).index.shape == (2, 0) + + +def test_interaction_vertex_and_genealogy(sample): + """Stored interaction vertices override differing roots, with local graph links.""" + raw = SuperaReader(file_keys=sample)[0] + raw["inter"].update(x=np.array([30.0]), y=np.array([60.0]), z=np.array([90.0])) + raw["particles"]["parent_id"][3] = 10 + raw["particles"]["ancestor_id"][3] = 10 + raw["particles"]["geant4_parent_trackid"] = np.arange(5) + 200 + raw["particles"]["geant4_parent_pdg"] = np.full(5, 211) + labels = SuperaClusterLabelParser("float32")(raw).particles + np.testing.assert_allclose(labels["vertex"], np.tile([10, 20, 30], (5, 1))) + particles = SuperaParticleParser("float32")(raw) + assert particles[3].interaction_primary == 0 + assert particles[3].ancestor_track_id == 10 + assert particles[3].parent_track_id == 203 + assert particles[3].parent_pdg_code == 211 + np.testing.assert_array_equal(particles[0].children_id, [3]) + np.testing.assert_array_equal( + SuperaParticleGraphParser("float32")(raw).index, [[0], [3]] + ) + # A nonfinite stored vertex falls back to valid ancestry. + raw.pop("_supera_prepared") + raw["inter"]["x"][:] = np.nan + labels = SuperaClusterLabelParser("float32")(raw).particles + np.testing.assert_allclose(labels["vertex"][0], [0.5, 0.5, 0.5]) + + +def check_truth_round_trip(events, path): + """Build all truth levels in cm and retain their associations through HDF5. + + Parameters + ---------- + events : list[dict] + Parsed events with the complete native schema and source provenance. + path : pathlib.Path + Temporary output HDF5 file. + + Returns + ------- + dict + Unwrapped truth products used for physical validation by callers. + """ + batch = CollateAll(data_keys=events[0])(events) + truth = Unwrapper()(batch) + original = copy.deepcopy(truth) + BuildManager(True, True, True, mode="truth", units="cm")(truth) + keys = [ + "meta", + "event_info", + "source_ids", + "points_label", + "depositions_label", + "truth_fragments", + "truth_particles", + "truth_interactions", + ] + writer = HDF5Writer(file_name=str(path), keys=keys, format_version=2) + writer(truth, {}) + writer.finalize() + writer.close() + reader = HDF5Reader(file_keys=str(path)) + for index, event in enumerate(events): + restored = reader[index] + BuildManager(True, True, True, mode="truth", units="cm")(restored) + # BuildManager converts parsed truth to cm; voxel centers use +0.5. + expected = event["meta"].to_cm( + original["clust_label"][index].coords, center=True + ) + np.testing.assert_allclose(restored["points_label"], expected) + assert np.isclose( + restored["depositions_label"].sum(), event["data"].features.sum() + ) + for key in ("truth_fragments", "truth_particles", "truth_interactions"): + assert len(restored[key]) == len(truth[key][index]) + for before, after in zip(truth[key][index], restored[key]): + assert after.units == "cm" + assert before.id == after.id + np.testing.assert_allclose(before.points, after.points) + for field in ( + "parent_id", + "children_id", + "orig_id", + "pid", + "is_primary", + "vertex", + "energy_deposit", + "fragment_ids", + "particle_ids", + ): + if hasattr(before, field): + np.testing.assert_allclose( + getattr(before, field), getattr(after, field) + ) + for particle in restored["truth_particles"]: + source = event["particles"][particle.orig_id] + assert particle.pid == source.pid + assert particle.is_primary == (source.interaction_primary > 0) + for particle in truth["truth_particles"][index]: + for fragment in particle.fragments: + assert fragment.particle_id == particle.id + for interaction in truth["truth_interactions"][index]: + for particle in interaction.particles: + assert particle.interaction_id == interaction.id + return truth + + +def test_truth_round_trip(sample, tmp_path): + """Sparse IDs, multiple events and empty truth survive the full conversion path.""" + import h5py + + with h5py.File(sample, "r+") as file: + for row in (3, 8): + file["particles/parent_id"][row] = 10 + file["particles/ancestor_id"][row] = 10 + ds = SuperaDataset(file_keys=sample, schema=FULL_SCHEMA, dtype="float32") + truth = check_truth_round_trip([ds[i] for i in (0, 2, 1)], tmp_path / "truth.h5") + # Source energy is per stored particle; grouping must count each deposit once. + assert truth["truth_particles"][0][0].energy_deposit == 9 + np.testing.assert_array_equal(truth["truth_particles"][0][0].children_id, [1]) + assert truth["truth_particles"][0][1].parent_id == 0 + assert len(truth["truth_particles"][1]) == 0 + + +def downstream_batch(sample): + """Create two nodes per nonempty event, separated by an empty event. + + Parameters + ---------- + sample : str + Synthetic Supera file path. + + Returns + ------- + tuple[dict, IndexBatch] + Collated labels and fragment nodes with event-local particle references. + """ + from spine.data import IndexBatch + + ds = SuperaDataset(file_keys=sample, schema=FULL_SCHEMA, dtype="float32") + batch = CollateAll(data_keys=ds.data_keys)([ds[0], ds[2], ds[1]]) + clusts = IndexBatch( + [np.array([0, 1]), np.array([2]), np.array([5, 6]), np.array([7])], + counts=[2, 0, 2], + spans=[5, 0, 5], + single_counts=[2, 1, 2, 1], + ) + return batch, clusts + + +def test_closest_primary_batch(sample): + """Closest-primary selection must use local table rows, not sparse source IDs.""" + from spine.cluster.label import ( + get_cluster_closest_label_batch, + get_cluster_closest_primary_label_batch, + ) + from spine.data import TensorBatch + + batch, clusts = downstream_batch(sample) + primary = TensorBatch(np.ones(4, dtype=int), counts=[2, 0, 2]) + for select in ( + get_cluster_closest_label_batch, + get_cluster_closest_primary_label_batch, + ): + args = (batch["clust_label"], batch["coord_label"], clusts, primary) + selected = ( + select(*args, default=[0, 0]) + if select == get_cluster_closest_label_batch + else select(*args) + ) + np.testing.assert_array_equal(selected.numpy_tensor(), [1, 0, 1, 0]) + # A fragment's PID/interaction-primary describe its host, not its own electron. + assert ( + batch["clust_label"].particle_field("pid")[0][2] + == batch["clust_label"].particle_field("pid")[0][0] + ) + + +def test_orientation_batch(sample): + """Orientation targets follow particle-table offsets across empty batch entries.""" + import pytest + + torch = pytest.importorskip("torch") + from spine.data import TensorBatch + from spine.model.grappa.loss.node_orient import NodeOrientLoss + + batch, clusts = downstream_batch(sample) + starts = TensorBatch( + torch.tensor( + [[0.5, 0.5, 0.5], [2.5, 0.5, 0.5], [1.5, 0.5, 0.5], [2.5, 0.5, 0.5]] + ), + [2, 0, 2], + ) + ends = TensorBatch( + torch.tensor( + [[1.5, 0.5, 0.5], [2.5, 0.5, 0.5], [0.5, 0.5, 0.5], [2.5, 0.5, 0.5]] + ), + [2, 0, 2], + ) + target = NodeOrientLoss().materialize_target( + clust_label=batch["clust_label"].to_tensor(), + coord_label=batch["coord_label"].to_tensor(), + clusts=clusts, + start_points=starts, + end_points=ends, + ) + np.testing.assert_array_equal(target["target"].numpy_tensor(), [1, -1, 0, -1]) + np.testing.assert_array_equal( + target["valid"].numpy_tensor(), [True, False, True, False] + ) + + +def test_absorbed_fragment_uses_host_interaction(sample): + """A deposit absorbed from another interaction follows its host's training labels.""" + raw = SuperaReader(file_keys=sample)[0] + raw["particles"]["interaction_id"][2] = 1 + raw["inter"].update( + interaction_id=np.array([0, 1]), + x=np.array([30.0, 300.0]), + y=np.array([60.0, 600.0]), + z=np.array([90.0, 900.0]), + ) + labels = SuperaClusterLabelParser("float32")(raw).particles + assert labels["interaction"][2] == 0 + assert labels["pid"][2] == labels["pid"][0] + np.testing.assert_allclose(labels["vertex"][2], [10, 20, 30]) + particle = SuperaParticleParser("float32")(raw)[2] + assert particle.interaction_id == 0 + np.testing.assert_allclose(particle.ancestor_position, [10, 20, 30]) + + +def test_example_schema_is_complete(sample): + """The documented configuration provides all products without parser overrides.""" + from pathlib import Path + + from spine.config import load_config_file + + config = load_config_file( + str(Path(__file__).resolve().parents[3] / "config/io/supera.yaml") + ) + schema = config["io"]["loader"]["dataset"]["schema"] + assert schema == FULL_SCHEMA + event = SuperaDataset(file_keys=sample, schema=schema, dtype="float32")[0] + BuildManager(True, True, True, mode="truth", units="cm")(event) + assert len(event["truth_particles"]) == 3 From e00f194030549bd6cd57cdb29c34d2909c75aeb8 Mon Sep 17 00:00:00 2001 From: Francois Drielsma Date: Tue, 3 Nov 2026 14:20:26 -0800 Subject: [PATCH 3/5] Fix Supera documentation inventory and example CI checks --- config/io/supera.yaml | 34 ------------------- docs/source/api/io.rst | 18 ++++++++-- test/test_io/test_supera/test_supera_truth.py | 13 ++++--- 3 files changed, 25 insertions(+), 40 deletions(-) delete mode 100644 config/io/supera.yaml diff --git a/config/io/supera.yaml b/config/io/supera.yaml deleted file mode 100644 index 51a44fb6..00000000 --- a/config/io/supera.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Native pysupera 4.1 input for generic-chain training and truth conversion. -# data contains truth-deposit energy (MeV), not measured pixel charge. -# Merge this io block into a training configuration; no conversion is required. -io: - loader: - batch_size: 1 - num_workers: 0 - shuffle: true - dataset: - provider: supera - file_keys: supera_pixel_j000000-000004.h5 - schema: - data: - provider: tensor - seg_label: - provider: segmentation - clust_label: - provider: cluster_label - ppn_label: - provider: particle_points - coord_label: - provider: particle_coords - graph_label: - provider: particle_graph - particles: - provider: particle - meta: - provider: meta - source_ids: - provider: source_ids - event_info: - provider: event_info - collate_fn: - provider: all diff --git a/docs/source/api/io.rst b/docs/source/api/io.rst index 289a4f48..1bd829db 100644 --- a/docs/source/api/io.rst +++ b/docs/source/api/io.rst @@ -60,7 +60,7 @@ productions. No pysupera or LArCV runtime is required. io: loader: - batch_size: 1 + minibatch_size: 1 num_workers: 0 dataset: provider: supera @@ -188,7 +188,7 @@ sum can differ from its attributed source-fragment energy. Truth conversion ~~~~~~~~~~~~~~~~ -Use the complete schema above (also in ``config/io/supera.yaml``) with: +Use the complete schema above with: .. code-block:: yaml @@ -300,6 +300,20 @@ are available on ``main``; there is no dependency on UResNetMulti. parse.supera.SuperaSourceIDsParser parse.supera.SuperaEventInfoParser +Implementation modules +~~~~~~~~~~~~~~~~~~~~~~ + +.. autosummary:: + :toctree: generated + + dataset.supera + read.supera + parse.supera.base + parse.supera.cluster + parse.supera.metadata + parse.supera.particle + parse.supera.tensor + Entry Filtering --------------- diff --git a/test/test_io/test_supera/test_supera_truth.py b/test/test_io/test_supera/test_supera_truth.py index 6de56909..c3f05b69 100644 --- a/test/test_io/test_supera/test_supera_truth.py +++ b/test/test_io/test_supera/test_supera_truth.py @@ -318,12 +318,17 @@ def test_absorbed_fragment_uses_host_interaction(sample): def test_example_schema_is_complete(sample): """The documented configuration provides all products without parser overrides.""" from pathlib import Path + from textwrap import dedent - from spine.config import load_config_file + import yaml - config = load_config_file( - str(Path(__file__).resolve().parents[3] / "config/io/supera.yaml") - ) + # Exercise the published schema directly, without a standalone config file. + doc_path = Path(__file__).resolve().parents[3] / "docs/source/api/io.rst" + section = doc_path.read_text().split("Native pysupera input\n", 1)[1] + example = section.split(".. code-block:: yaml\n\n", 1)[1].split("\n\n", 1)[0] + config = yaml.safe_load(dedent(example)) + assert config["io"]["loader"]["minibatch_size"] == 1 + assert "batch_size" not in config["io"]["loader"] schema = config["io"]["loader"]["dataset"]["schema"] assert schema == FULL_SCHEMA event = SuperaDataset(file_keys=sample, schema=schema, dtype="float32")[0] From 4e0587691b6e0b5e36b4f8b237fd163559df6161 Mon Sep 17 00:00:00 2001 From: Francois Drielsma Date: Mon, 9 Nov 2026 09:21:30 -0800 Subject: [PATCH 4/5] Replace direct pysupera loading with PIMM detector-hit input --- .github/workflows/ci.yml | 2 +- docker/spine/Dockerfile | 6 +- docs/source/api/io.rst | 345 +++++-------- docs/source/api/supera_format_findings.rst | 30 +- pyproject.toml | 8 +- src/spine/io/dataset/__init__.py | 4 +- src/spine/io/dataset/{supera.py => pimm.py} | 26 +- .../io/parse/{supera => pimm}/__init__.py | 2 +- src/spine/io/parse/pimm/base.py | 281 +++++++++++ .../io/parse/{supera => pimm}/cluster.py | 8 +- .../io/parse/{supera => pimm}/metadata.py | 38 +- .../io/parse/{supera => pimm}/particle.py | 78 ++- src/spine/io/parse/{supera => pimm}/tensor.py | 20 +- src/spine/io/parse/supera/base.py | 456 ------------------ src/spine/io/read/__init__.py | 4 +- src/spine/io/read/pimm.py | 159 ++++++ src/spine/io/read/supera.py | 267 ---------- .../{test_supera => test_pimm}/__init__.py | 0 test/test_io/test_pimm/conftest.py | 127 +++++ test/test_io/test_pimm/coverage.ini | 13 + test/test_io/test_pimm/test_parsers.py | 211 ++++++++ .../test_pimm_chain.py} | 42 +- .../test_pimm_model.py} | 18 +- test/test_io/test_pimm/test_reader.py | 159 ++++++ test/test_io/test_pimm/truth_helpers.py | 89 ++++ test/test_io/test_supera/conftest.py | 152 ------ test/test_io/test_supera/coverage.ini | 13 - test/test_io/test_supera/test_supera.py | 319 ------------ test/test_io/test_supera/test_supera_truth.py | 336 ------------- 29 files changed, 1296 insertions(+), 1917 deletions(-) rename src/spine/io/dataset/{supera.py => pimm.py} (70%) rename src/spine/io/parse/{supera => pimm}/__init__.py (59%) create mode 100644 src/spine/io/parse/pimm/base.py rename src/spine/io/parse/{supera => pimm}/cluster.py (86%) rename src/spine/io/parse/{supera => pimm}/metadata.py (60%) rename src/spine/io/parse/{supera => pimm}/particle.py (77%) rename src/spine/io/parse/{supera => pimm}/tensor.py (70%) delete mode 100644 src/spine/io/parse/supera/base.py create mode 100644 src/spine/io/read/pimm.py delete mode 100644 src/spine/io/read/supera.py rename test/test_io/{test_supera => test_pimm}/__init__.py (100%) create mode 100644 test/test_io/test_pimm/conftest.py create mode 100644 test/test_io/test_pimm/coverage.ini create mode 100644 test/test_io/test_pimm/test_parsers.py rename test/test_io/{test_supera/test_supera_chain.py => test_pimm/test_pimm_chain.py} (81%) rename test/test_io/{test_supera/test_supera_model.py => test_pimm/test_pimm_model.py} (92%) create mode 100644 test/test_io/test_pimm/test_reader.py create mode 100644 test/test_io/test_pimm/truth_helpers.py delete mode 100644 test/test_io/test_supera/conftest.py delete mode 100644 test/test_io/test_supera/coverage.ini delete mode 100644 test/test_io/test_supera/test_supera.py delete mode 100644 test/test_io/test_supera/test_supera_truth.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb0eea07..cd166d4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,7 +226,7 @@ jobs: -w /workspace \ "${COVERAGE_IMAGE}" \ bash -lc ' - python -m pip install --root-user-action=ignore -e . pytest pytest-cov coverage[toml] + python -m pip install --root-user-action=ignore -e ".[pimm]" pytest pytest-cov coverage[toml] mkdir -p /tmp/spine-sitecustomize printf "import h5py\n" > /tmp/spine-sitecustomize/sitecustomize.py NUMBA_DISABLE_JIT=1 PYTHONPATH="/tmp/spine-sitecustomize:${PYTHONPATH:-}" pytest test \ diff --git a/docker/spine/Dockerfile b/docker/spine/Dockerfile index 770428fb..c3b52dbe 100644 --- a/docker/spine/Dockerfile +++ b/docker/spine/Dockerfile @@ -184,9 +184,9 @@ RUN wget -O "${ICARUS_PHOTONLIB_PATH}" "${ICARUS_PHOTONLIB_URL}" && \ COPY pyproject.toml MANIFEST.in README.md ./ COPY src/ ./src/ -# Install SPINE with the optional visualization stack. The compiled model -# dependencies are installed explicitly above to keep their versions aligned. -RUN pip install --no-cache-dir --no-build-isolation .[viz] +# Install SPINE with PIMM readers and the optional visualization stack. +# Compiled model dependencies are installed above to keep versions aligned. +RUN pip install --no-cache-dir --no-build-isolation ".[viz,pimm]" # Verify that the default container environment and the explicit fallback setup # recipe both provide the compiled dependency imports SPINE needs at runtime. diff --git a/docs/source/api/io.rst b/docs/source/api/io.rst index 1bd829db..2706d1f4 100644 --- a/docs/source/api/io.rst +++ b/docs/source/api/io.rst @@ -44,17 +44,29 @@ File Readers :toctree: generated read.HDF5Reader - read.SuperaReader + read.PIMMReader read.LArCVReader read.CacheReader -Native pysupera input ---------------------- +.. _pimm-input: -Use ``provider: supera`` for pysupera **4.1.0** files. This is a separate -format from SPINE HDF5 V1/V2; ``provider: hdf5`` does not read these files. -Install ``spine[supera]`` for the HDF5 compression filters used by upstream -productions. No pysupera or LArCV runtime is required. +PIMM detector input +------------------- + +Use ``provider: pimm`` for paired JAXTPC ``hits`` and pysupera-derived ``labl`` +files in the PIMM layout. ``PIMMReader`` delegates decoding, event alignment, +per-hit overrides and particle-ID normalization to ``pimm_data.JAXTPCDataset``. +SPINE consumes its untransformed NumPy output and retains its own parsers, +collation and augmentation. There is no direct pysupera HDF5 reader or +truth-deposit input fallback. + +Install ``spine[pimm]``. The optional extra includes PyTorch, ``pimm-data`` and +HDF5 compression filters; importing base SPINE does not import ``pimm-data``. +The Docker image installs this extra. The provisional dependency is pinned to +``doraemon-final-layout`` commit ``23dc94f3ac1134936891b281e4408a975d0097ed``; +an upstream release is required before this integration becomes official. +See the `upstream examples +`_. .. code-block:: yaml @@ -63,256 +75,129 @@ productions. No pysupera or LArCV runtime is required. minibatch_size: 1 num_workers: 0 dataset: - provider: supera - file_keys: supera_pixel_j000000-000004.h5 + provider: pimm + data_root: /path/to/jaxtpc_pixel + dataset_name: sim_pixel + runs: "j*/*" schema: data: {provider: tensor} seg_label: {provider: segmentation} clust_label: {provider: cluster_label} - ppn_label: {provider: particle_points} - coord_label: {provider: particle_coords} - graph_label: {provider: particle_graph} particles: {provider: particle} + coord_label: {provider: particle_coords} + ppn_label: {provider: particle_points} meta: {provider: meta} - source_ids: {provider: source_ids} + graph_label: {provider: particle_graph} event_info: {provider: event_info} - collate_fn: {provider: all} - -``SuperaDataset`` reuses the HDF5 dataset's parsing, collation and augmentation -interface with a format-specific parser registry. ``SuperaReader`` supports -multiple files, ordinary entry selection, source-file provenance and projected -reads. Its default raw products are ``points``, ``particles``, ``inter``, -``events`` and ``metadata``. Particle and interaction point ranges are rebased -to the event's point array. Interaction particle ranges are likewise rebased. -The ``event_info`` parser preserves every stored event/input/job identity as a -named integer tensor, including ``event_id``, ``input``, ``input_event``, -``jaxtpc_entry``, ``job_event`` and ``job_id`` in the sample. It does not invent -run/subrun numbers. The raw identities remain available as ``events``. - -Physical semantics -~~~~~~~~~~~~~~~~~~ - -The implementation was checked against `pysupera commit dab92d8 -`_, -in particular ``_run.py``, ``layout.py``, ``io_v3.py``, ``provenance.py`` and -``torchdata.py``: - -* With JAXTPC input, ``points/flat`` contains **truth energy deposits**, not - measured hits. Coordinates are mm, ``t`` is microseconds, ``dE`` is MeV, - ``dX`` is cm, angles are radians and ``p`` is MeV/c. ``hit_energy=charge`` - describes the hit-based partitioning input; it does not change ``dE`` into - charge. Measured hit coordinates and charge require the separate JAXTPC file. -* ``groups`` is indexed by JAXTPC deposit group, concatenated across volumes. - Its rows do not align with ``points/flat``. In the supplied 1,000-event - production there are 54,863,490 groups and 31,871,012 point rows. - ``volume_offsets`` identifies each volume's start in the event's group array. -* Fragment representatives satisfy ``frag_id == id``. Their half-open - ``frag_pc_start:frag_pc_end`` ranges partition the point rows. Instance ranges - also partition the points, with non-LE fragments preceding the LE portion - starting at ``inst_pc_le_start``. ``frag_inst_id`` joins each fragment to - its instance. The reader never assumes group/point alignment. -* The writer retains sparse particle IDs. Despite the upstream reader module's - row-index wording, these must be resolved through an ID-to-row map. SPINE - particle IDs, cluster-sidecar ``particle`` IDs, ancestor links, graph nodes - and PPN references are all event-local particle-table rows. ``source_ids`` - preserves the sparse source ID for each row, without shifting it in a batch. -* Semantic codes map explicitly: pysupera shower/track/delta/Michel/LE - ``1/2/3/4/5`` become SPINE ``0/1/3/2/4``. Non-LE points use the instance's - semantic class; LE fragments retain LE even inside a track or shower. - Unknown point semantics are rejected by the five-class training parsers. -* Particle ``start_*``/``end_*`` describe Geant4 creation/destruction. - Instance ``first_*``/``last_*`` describe the first/last collected steps, - subject to ``kinematics_steps`` (``visible`` in the sample). Kinematic - positions are mm and times ns. Showers have no last endpoint. - -Aligned training products -~~~~~~~~~~~~~~~~~~~~~~~~~ - -All parsers share one event preparation. Coordinates are rasterized with the -file's ``voxel_size`` and an origin of zero, then shifted by an integer number -of cells to make event coordinates nonnegative. ``Meta`` records the inverse -transform in cm. Every spatial parser accepts ``voxel_size_mm`` (scalar or three-vector) -and ``origin_mm``; all spatial parsers in a dataset must use the same grid settings. -The identity-only ``event_info`` parser does not take grid arguments. -An explicit size is required when the file does not supply one. These settings -also permit rasterizing deposit-mode files. Upstream voxelization can use a -per-particle origin, so stored centers are rasterized rather than assumed to -already be integer coordinates on a common grid. - -Overlapping contributions sum their energy. Label ownership follows upstream -priority (track, shower, delta, Michel, LE), then earliest deposit time, then -source row order. The same winning contribution selects semantic, fragment -and instance labels. LE deposits are retained. ``ClusterLabelData`` stores -fragment IDs and fragment-row associations, with instance IDs in ``group``. -Unprovided neutrino identity is left at ``-1``. No energy-to-charge conversion -is performed. - -``coord_label`` is a separate, row-aligned ``particle_coords`` product for -GrapPA. Its ``TensorData`` schema has ``start`` and ``end`` coordinate groups, -plus ``time`` (ns) and ``shape`` features. Every stored particle gets a row, -including particles with no retained voxels. Instance rows use stored collected -step endpoints. Where these are missing, fragment rows use earliest/latest -collected deposit positions; instance rows use their non-LE deposit range, -or all deposits when that range is empty. These fallback positions are derived -from voxelized deposits, **not exact Geant4 step endpoints**. Non-track ends -repeat their starts. Rows without usable endpoints retain NaN coordinates and -times. Deposited rows have finite fallback coordinates; no creation/destruction -position is substituted. The same endpoint resolution populates parsed particle -``first_step``/``last_step`` fields. - -PPN targets use finite stored instance starts and track ends only, exclude LE, -unknown classes and nuclear fragments, and omit points outside the event -image. They reference the complete event particle table but cannot replace -``coord_label`` for closest-primary selection or track orientation. - -The particle parser fills PID, group/interaction-primary flags and ancestry -consistently with cluster supervision. PID and interaction-primary labels are -propagated from each fragment's host instance, preventing daughter-electron -votes from changing an instance's PID. A group primary is its representative; -an interaction primary is a host whose redirected stored parent is itself. -This is the file's stored-root convention, not an independently inferred -generator-primary definition. Host-instance interaction IDs label absorbed -fragments, whose own source interaction may differ. Stored interaction vertices -supply a common ``ancestor_position`` and cluster ``vertex`` for all members. -When unavailable, the stored ancestor's creation position is used; missing -ancestry remains NaN rather than indexing row ``-1``. - -Individual ``pc_start:pc_end`` ranges do not cover all deposits in this file: -absorbed particles may have been omitted from the stored table. Fragment ranges -do partition the point cloud. SPINE assigns each fragment's summed dE once, -to its representative row, and zero to other rows. Truth particle construction -then sums these fragment contributions without double-counting. This energy -is independent of overlap-winner assignment; a truth object's voxel deposition -sum can differ from its attributed source-fragment energy. - -Truth conversion -~~~~~~~~~~~~~~~~ - -Use the complete schema above with: + +For separately located shards, omit ``runs`` and provide ``files`` with +``hits`` and ``labl`` paths or glob patterns. The upstream loader requires +matching task and event identities and rejects partial pairs. Standard SPINE +entry selection (``n_entry``, ``n_skip``, ``entry_list``) is applied afterward. +Do not supply the old ``supera_pixel_*.h5`` as a ``labl`` file: prepare it with +upstream ``python -m pimm_data.supera_labl SUPERA HITS OUT`` first, or use the +published final-layout corpus. Conversion is an explicit preparation step, +never a hidden side effect of training. + +Coordinates, charge and supervision +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The 3D parsers currently require **pixel hits**. Upstream can read wire hits, +but a wire/tick pair does not define a unique 3D point; these parsers reject +wire input instead of interpreting its indexes as detector coordinates. + +Pixel indexes ``(py, pz, tick)`` are converted using ``supera_pixel_geometry`` +from the ``labl/config`` attributes. The transform uses pixel centers and +nominal drift coordinates, including the stored reference tick and drift +sign. ``supera_hit_x_from=nominal`` and ``supera_hit_energy=charge`` are required. +No interaction-time correction is inferred from truth. + +All spatial parsers share ``voxel_size_mm`` (default 3) and ``origin_mm`` +(default zero). Each event's integer grid is translated to nonnegative +coordinates; the shared ``Meta`` retains its absolute position in cm. +Conflicting parser grids are rejected. + +* The tensor contains **summed hit charge**, in the source hit units, not MeV. + Signed charge is retained. Coincident hits share one voxel. +* Semantic labels come directly from ``segment_motif``: shower, track, Michel, + delta, LE. Overlap winners use track, shower, delta, Michel, LE priority, + followed by source row order; every voxel product shares that selection. +* Voxels without any labelled contribution are excluded by default because + generic SPINE segmentation requires targets in 0--4. Charge from all hits + in retained voxels is preserved. Set ``include_unlabeled: true`` on every + spatial parser to retain target ``-1`` for inspection, not generic training. + LE remains class 4 with its upstream ownership references + (``le_is_stuff=False``); downstream objectives retain their normal selection. +* Fragment, instance, ancestor and graph references use event-local rows of + the PIMM particle table. SPINE collation applies the batch offsets. +* Coordinate labels contain one start/end pair per particle-table row. Stored + collected-step positions use the shared transform; missing endpoints remain + NaN. No deposit-derived or track-creation substitute is used. PPN includes + finite, in-image instance starts and track ends. +* Particle PID is mapped from PDG to SPINE's classes. Primary flags and + genealogy use the upstream table; interaction vertices use the host + instance's interaction. Missing ancestry is never indexed through ``-1``. +* Particle deposited energy is unavailable from these hit/label products and + remains NaN. Initial energy and momentum retain MeV and MeV/c. Generator + neutrino information is unavailable. The ``event_info`` product preserves + the upstream event ID when present and the source entry, without invented + run/subrun values. + +Truth construction uses the complete schema above with: .. code-block:: yaml - base: - unwrap: true build: mode: truth units: cm - fragments: true - particles: true - interactions: true -No model is required for truth-only construction. ``meta`` exposes the shared -``Meta`` at top level, so collation, unwrapping and ``BuildManager`` can convert -voxel coordinates to detector centimetres. Voxel centers use the half-cell -shift; particle endpoints and interaction vertices retain their continuous -physical positions. ``graph_label`` supplies directed stored-parent edges; -truth building contracts invisible ancestors and restores parent/child links -between retained particles. Without this product, those output genealogy links -are not restored. The standard SPINE HDF5 V2 writer/reader preserves these -products and associations; load truth objects through ``BuildManager`` to -reattach their long-form point arrays. - -Unavailable neutrino identity, unknown PID and missing relationship IDs use -``-1``. Missing PDG codes default to SPINE's ``INVALID_PDG`` (0); the source -Geant4-parent PDG sentinel (-1) is retained when present. Absent creation -process names are empty strings. Missing positions, times and travel distance -are NaN. Neutrino interaction records, measured charge, generator-process -strings and exact endpoints of individual fragments cannot be recovered from -this file. The reader does not synthesize them. See the -:doc:`author-facing format report ` for measured format -limitations and proposed upstream changes. +Known limitations and validation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. toctree:: :hidden: supera_format_findings -``groups`` and ``hit_labels/volume0/plane0`` (or another explicit plane path) -can be requested through ``SuperaReader(keys=...)``. They are never read by -the training parsers or by default. Even explicitly requested hit labels are -sliced by event offsets; no whole-plane label array is materialized. - -Validation -~~~~~~~~~~ - -The synthetic tests cover sparse IDs, fragment/instance distinctions, LE, -semantic remapping, overlapping energy, empty events, multi-file selection, -absolute-range rebasing, pickle round trips and projected hit reads. Error-path -tests also cover invalid grids, corrupt relationships, nonfinite values, -missing compression support and endpoint filtering. The dedicated coverage -configuration requires 100% line and branch coverage of the Supera modules, -without excluding defensive or optional-dependency paths: - -.. code-block:: bash - - PYTHONPATH=src pytest -q test/test_io/test_supera \ - --cov=src/spine --cov-config=test/test_io/test_supera/coverage.ini \ - --cov-report=term-missing +This integration replaces the earlier truth-deposit prototype. Its old +training results do not establish detector-charge performance. Missing +fragment endpoints can reduce primary/orientation supervision; drift timing +can also displace nominal hit coordinates relative to true endpoints. +The reader must not silently repair either using truth deposits. -Run the real-file forward/backward test in the model runtime with: +Synthetic tests cover alignment, charge conservation, ignored labels, empty +clouds, missing endpoints, event selection, worker serialization, batching and +truth HDF5 round trips. Optional production checks use paired ``hits``/``labl`` +directories via ``SPINE_PIMM_ROOT`` and assert nonzero supervision and gradients +for segmentation, PPN, Graph-SPICE and shower/track/interaction GrapPA. Their +availability is not a claim that every objective has eligible targets in every +production. .. code-block:: bash - docker run --rm --platform linux/amd64 \ - -e PYTHONPATH=/workspace/src -e OMP_NUM_THREADS=1 \ - -e SPINE_SUPERA_SAMPLE=/workspace/supera_pixel_j000000-000004.h5 \ - -v "$PWD:/workspace" -w /workspace \ - ghcr.io/deeplearnphysics/spine:latest bash -lc \ - 'pip install hdf5plugin && pytest -q -s test/test_io/test_supera' - -The first sample event yields 25,537 unique voxels and 120 PPN targets. -The opt-in integration suite exercises these generic-chain components: - -* UResNet segmentation and PPN forward/backward on the complete first event, - with segmentation-stage cache publication and readback. -* Graph-SPICE fragmentation forward/backward on the same aligned voxel labels. -* Fragment-graph feature/target materialization and cache readback, followed by - shower GrapPA primary/grouping and track GrapPA grouping forward/backward. -* Particle-graph feature/target materialization and cache readback, followed by - interaction GrapPA PID, closest-primary, orientation and grouping - forward/backward. -* Truth fragment, particle and interaction construction for events 0 and 1, - followed by SPINE HDF5 V2 writing, reading and association rebuilding in cm. - -GrapPA checks use two complete events with truth-defined input clusters and the -maintained generic configuration, reducing only hidden neural widths and -enabling closest-primary matching. They retain all configured objectives and -assert valid, nonzero supervised counts and gradients for each. These checks -validate each learned component and its cache contract; they do not measure -reconstruction quality or assert that an untrained end-to-end network produces -useful clusters. Synthetic tests additionally exercise sparse IDs larger than -the particle table, empty batch entries, missing endpoints, missing ancestors, -closest-primary matching and opposite track orientations across events. -The Supera modules retain 100% line and branch coverage. All exercised models -are available on ``main``; there is no dependency on UResNetMulti. - -.. autosummary:: - :toctree: generated - - parse.supera.SuperaTensorParser - parse.supera.SuperaSegmentationParser - parse.supera.SuperaClusterLabelParser - parse.supera.SuperaParticleParser - parse.supera.SuperaParticlePointsParser - parse.supera.SuperaParticleCoordsParser - parse.supera.SuperaParticleGraphParser - parse.supera.SuperaMetaParser - parse.supera.SuperaSourceIDsParser - parse.supera.SuperaEventInfoParser - -Implementation modules -~~~~~~~~~~~~~~~~~~~~~~ + PYTHONPATH=src pytest -q test/test_io/test_pimm + SPINE_PIMM_ROOT=/path/to/paired/data PYTHONPATH=src \ + pytest -q -s test/test_io/test_pimm .. autosummary:: :toctree: generated - dataset.supera - read.supera - parse.supera.base - parse.supera.cluster - parse.supera.metadata - parse.supera.particle - parse.supera.tensor + parse.pimm.PIMMTensorParser + parse.pimm.PIMMSegmentationParser + parse.pimm.PIMMClusterLabelParser + parse.pimm.PIMMParticleParser + parse.pimm.PIMMParticlePointsParser + parse.pimm.PIMMParticleCoordsParser + parse.pimm.PIMMParticleGraphParser + parse.pimm.PIMMMetaParser + parse.pimm.PIMMEventInfoParser + dataset.pimm + read.pimm + parse.pimm.base + parse.pimm.cluster + parse.pimm.metadata + parse.pimm.particle + parse.pimm.tensor Entry Filtering --------------- @@ -501,7 +386,7 @@ with raw input through the mixed dataset. dataset.LArCVDataset dataset.CacheDataset dataset.HDF5Dataset - dataset.SuperaDataset + dataset.PIMMDataset dataset.MixedDataset dataset.JointDataset diff --git a/docs/source/api/supera_format_findings.rst b/docs/source/api/supera_format_findings.rst index 57b7a023..744c99a5 100644 --- a/docs/source/api/supera_format_findings.rst +++ b/docs/source/api/supera_format_findings.rst @@ -4,7 +4,35 @@ pysupera 4.1 integration findings This report is prepared for the pysupera authors from the SPINE PR #191 integration. It distinguishes an incorrect documented contract from intentional loss of source detail and requests for a more explicit interchange schema. -It has not been submitted upstream. +The author responses below supersede the original open questions. + +.. note:: + + This is the historical investigation of the retired direct pysupera adapter. + The measurements remain useful, but descriptions of SPINE's deposit-based + energy attribution and endpoint fallbacks below do not describe the current + :ref:`PIMM integration `. + +Author clarification and integration direction +---------------------------------------------- + +The authors confirmed that the ``io_v3.py`` index docstring is outdated: +stored IDs are pre-filter in-memory particle indexes. They intend to change +these to stored row indexes. Own-particle point ranges describe deposits before +merging and are not expected to cover contributions from omitted particles. +Only LE contributions may merge across interaction boundaries. Self-parenting +is a supported primary criterion; ``geant4_parent_trackid == -1`` is the +preferred direct test. Fragment endpoint information is being added upstream. + +Crucially, detector training must read the separate JAXTPC hits, not +``points/flat``. The latter contains voxelized EDepSim energy in MeV; +``hit_energy=charge`` describes the hits and does not change that product. +Entry ``k`` of ``hit_labels/volumeV/plane0`` labels entry ``k`` of the matched +JAXTPC pixel hit array, with the source entry given by ``events/jaxtpc_entry``. + +SPINE is pivoting to a ``pimm`` provider using ``pimm-data`` for detector input +and pysupera for truth. The measurements below remain useful observations of +the original sample, but are not evidence of detector-charge training support. Reference production -------------------- diff --git a/pyproject.toml b/pyproject.toml index 484752c9..07b0f8fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,12 @@ Documentation = "https://spine.readthedocs.io/" "Bug Tracker" = "https://github.com/DeepLearnPhysics/spine/issues" [project.optional-dependencies] -# Compression filters used by native pysupera files -supera = ["hdf5plugin>=4"] +# Detector-hit readers and compression filters used by PIMM productions. +# Provisional doraemon-final-layout commit; require a release before adoption. +pimm = [ + "pimm-data @ git+https://github.com/OmarAlterkait/pimm-data.git@23dc94f3ac1134936891b281e4408a975d0097ed", + "hdf5plugin>=4", +] # Visualization dependencies viz = [ diff --git a/src/spine/io/dataset/__init__.py b/src/spine/io/dataset/__init__.py index b4cf2668..277c6270 100644 --- a/src/spine/io/dataset/__init__.py +++ b/src/spine/io/dataset/__init__.py @@ -15,7 +15,7 @@ from .joint import JointDataset from .larcv import LArCVDataset from .mixed import MixedDataset -from .supera import SuperaDataset +from .pimm import PIMMDataset __all__ = [ "CacheDataset", @@ -23,5 +23,5 @@ "JointDataset", "LArCVDataset", "MixedDataset", - "SuperaDataset", + "PIMMDataset", ] diff --git a/src/spine/io/dataset/supera.py b/src/spine/io/dataset/pimm.py similarity index 70% rename from src/spine/io/dataset/supera.py rename to src/spine/io/dataset/pimm.py index 0e33ef1f..94bd1f85 100644 --- a/src/spine/io/dataset/supera.py +++ b/src/spine/io/dataset/pimm.py @@ -1,4 +1,4 @@ -"""Native pysupera dataset using the common HDF5 parsing machinery.""" +"""PIMM detector-hit dataset using the common HDF5 parsing machinery.""" from __future__ import annotations @@ -7,22 +7,22 @@ from spine.config.factory import module_dict -from ..parse import supera -from ..read.supera import SuperaReader +from ..parse import pimm +from ..read.pimm import PIMMReader from .hdf5 import HDF5Dataset -__all__ = ["SuperaDataset"] +__all__ = ["PIMMDataset"] -class SuperaDataset(HDF5Dataset): - """Parse pysupera truth deposits into aligned SPINE training products.""" +class PIMMDataset(HDF5Dataset): + """Parse aligned PIMM hits and truth into aligned SPINE training products.""" - name: ClassVar[str] = "supera" - parser_dict = module_dict(supera) + name: ClassVar[str] = "pimm" + parser_dict = module_dict(pimm) def _build_reader( self, stage: str | None, stage_map: Mapping[str, str], kwargs: Mapping[str, Any] - ) -> SuperaReader: + ) -> PIMMReader: """Select the native reader and enforce a common voxelization grid. Parameters @@ -36,7 +36,7 @@ def _build_reader( Returns ------- - SuperaReader + PIMMReader Native reader projected onto the products required by the schema. Raises @@ -49,9 +49,11 @@ def _build_reader( # Every supervision product must use the same voxel coordinates. grids = {p.grid for p in self.parsers.values() if hasattr(p, "grid")} if len(grids) > 1: - raise ValueError("Supera parsers must use the same voxel size and origin.") + raise ValueError( + "PIMM parsers must use the same voxel size, origin and unlabelled policy." + ) # Reuse the parent dataset's schema-driven physical projection. - return SuperaReader( + return PIMMReader( keys=None if self.keys is None else tuple(self.keys), **kwargs ) diff --git a/src/spine/io/parse/supera/__init__.py b/src/spine/io/parse/pimm/__init__.py similarity index 59% rename from src/spine/io/parse/supera/__init__.py rename to src/spine/io/parse/pimm/__init__.py index 3833e0d2..2072a6c9 100644 --- a/src/spine/io/parse/supera/__init__.py +++ b/src/spine/io/parse/pimm/__init__.py @@ -1,4 +1,4 @@ -"""Parsers for native pysupera truth-deposit products.""" +"""Parsers for aligned PIMM detector hits and truth tables.""" from .cluster import * from .metadata import * diff --git a/src/spine/io/parse/pimm/base.py b/src/spine/io/parse/pimm/base.py new file mode 100644 index 00000000..72d58899 --- /dev/null +++ b/src/spine/io/parse/pimm/base.py @@ -0,0 +1,281 @@ +"""Shared PIMM hit voxelization and particle supervision.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.constants import PDG_TO_PID +from spine.data import Meta + +from ..base import ParserBase + + +def _shape(code: int | np.ndarray) -> np.ndarray: + """Map upstream one-based semantic codes to SPINE's five classes. + + Parameters + ---------- + code : int or np.ndarray + Upstream semantic codes; unsupported codes become -1. + + Returns + ------- + np.ndarray + SPINE class indexes with the same shape as the input. + """ + return np.array([-1, 0, 1, 3, 2, 4, -1])[np.clip(code, 0, 6)] + + +class PIMMParserBase(ParserBase): + """Share one hit grid and overlap selection across all training products. + + Pixel indexes are converted to nominal detector coordinates in mm, then + voxelized. Metadata uses cm. Charge is summed without energy conversion. + Label ties use track, shower, delta, Michel, LE, then source row order. + Voxels without labelled contributions are excluded by default. Charge + from every contribution is retained in a labelled voxel. Wire hits cannot + supply a unique 3D position. + """ + + def __init__( + self, + dtype: str, + voxel_size_mm: float | tuple[float, ...] = 3.0, + origin_mm: tuple[float, ...] = (0.0, 0.0, 0.0), + include_unlabeled: bool = False, + ) -> None: + """Configure the common detector grid. + + Parameters + ---------- + dtype : str + Output floating-point dtype. + voxel_size_mm : float or sequence[float], default 3 + Three physical voxel dimensions in mm. + origin_mm : sequence[float], default (0, 0, 0) + Detector grid origin in mm. + include_unlabeled : bool, default False + Retain voxels with target -1 for inspection. Generic SPINE + segmentation training requires these voxels to be excluded. + """ + super().__init__(dtype, pimm_event="pimm") + self.size = tuple(np.broadcast_to(np.asarray(voxel_size_mm, float), (3,))) + self.origin = tuple(np.asarray(origin_mm, float)) + if len(self.origin) != 3 or not np.isfinite(self.origin).all(): + raise ValueError("origin_mm must contain three finite values.") + if not np.isfinite(self.size).all() or min(self.size) <= 0: + raise ValueError("voxel_size_mm must be finite and positive.") + self.include_unlabeled = include_unlabeled + self.grid = (self.size, self.origin, include_unlabeled) + + def prepare(self, trees: dict[str, Any]) -> dict[str, Any]: + """Cache an aligned charge image, labels and particle-table references. + + Parameters + ---------- + trees : dict + Reader products containing the nested PIMM event. + + Returns + ------- + dict + Voxel coordinates, charge, truth references and shared metadata. + """ + cache = trees.setdefault("_pimm_prepared", {}) + if self.grid in cache: + return cache[self.grid] + raw = trees["pimm"] + hits, table = raw["hits"], raw["particle"] + points = self._coordinates(hits, raw["metadata"]) + values = np.asarray(hits["energy"]).reshape(-1) + semantic = np.asarray(hits["segment_motif"]).reshape(-1) + fragment = np.asarray(hits["instance_fragment"]).reshape(-1) + instance = np.asarray(hits["instance_particle"]).reshape(-1) + if any( + len(col) != len(points) for col in (values, semantic, fragment, instance) + ): + raise ValueError("PIMM hit and label row counts differ.") + if not np.isfinite(values).all() or not np.isfinite(points).all(): + raise ValueError("Nonfinite PIMM hit charge or coordinates.") + if np.any((semantic < -1) | (semantic > 4)): + raise ValueError("Invalid PIMM semantic labels.") + for refs in (fragment, instance): + if np.any((refs < -1) | (refs >= len(table["parent"]))): + raise ValueError("Invalid PIMM particle-table reference.") + + event = self._voxelize(points, values, semantic) + winner = event.pop("winner") + if not self.include_unlabeled: + keep = semantic[winner] >= 0 + winner = winner[keep] + event["coords"] = event["coords"][keep] + event["energy"] = event["energy"][keep] + event.update( + shape=semantic[winner], + fragment=fragment[winner], + instance=instance[winner], + columns=table, + ) + event["labels"] = self._particle_labels(event, raw["interaction"]) + cache[self.grid] = event + return event + + def _voxelize( + self, points: np.ndarray, values: np.ndarray, semantic: np.ndarray + ) -> dict[str, Any]: + """Sum coincident charge and select one shared label owner per voxel. + + Parameters + ---------- + points : np.ndarray + Physical hit coordinates in mm. + values : np.ndarray + Signed hit charge in the source units. + semantic : np.ndarray + Class labels used to resolve overlaps, including unknown (-1). + + Returns + ------- + dict + Sparse coordinates, summed charge, winning rows and cm metadata. + """ + size = np.asarray(self.size) + cells = np.floor((points - self.origin) / size).astype(np.int64) + lower_cell = cells.min(axis=0) if len(points) else np.zeros(3, dtype=np.int64) + coords, inverse = np.unique(cells - lower_cell, axis=0, return_inverse=True) + inverse = inverse.reshape(-1) + ranks = np.full(len(points), 5) + for rank, shape in enumerate((1, 0, 3, 2, 4)): + ranks[semantic == shape] = rank + order = np.lexsort((np.arange(len(points)), ranks, inverse)) + lower = (np.asarray(self.origin) + lower_cell * size) / 10.0 + count = coords.max(axis=0) + 1 if len(points) else np.ones(3, dtype=np.int64) + return { + "coords": coords, + "energy": np.bincount(inverse, weights=values), + "winner": ( + order[np.r_[True, np.diff(inverse[order]) != 0]] + if len(points) + else order + ), + "meta": Meta( + lower=lower, + upper=lower + count * size / 10.0, + size=size / 10.0, + count=count, + ), + } + + @staticmethod + def _coordinates(hits: dict, metadata: dict) -> np.ndarray: + """Convert pixel channel/tick indexes to nominal positions in mm. + + Parameters + ---------- + hits : dict + Upstream hit cloud in ``(py, pz, time)`` order. + metadata : dict + Per-volume pixel geometry copied by the PIMM label converter. + + Returns + ------- + np.ndarray + Detector ``(x, y, z)`` coordinates, one row per hit. + """ + if hits["readout_type"] != "pixel": + raise ValueError("PIMM 3D parsers require pixel hits; wire hits are 2D.") + geometry = metadata.get("pixel_geometry") + if geometry is None or metadata.get("hit_x_from") != "nominal": + raise ValueError( + "PIMM pixel input requires nominal pixel_geometry metadata." + ) + if metadata.get("hit_energy") != "charge": + raise ValueError("PIMM tensor input requires hit_energy=charge.") + coordinates = np.asarray(hits["coord"], dtype=float) + positions = np.empty_like(coordinates) + for plane_id, plane in enumerate(hits["planes"]): + volume = int(plane.split("_")[1]) + g = geometry[volume] + mask = np.asarray(hits["plane_id"]).reshape(-1) == plane_id + py, pz, ticks = coordinates[mask].T + positions[mask, 0] = ( + g["x_anode_mm"] + - g["drift_direction"] + * (ticks - g["reference_tick"]) + * g["mm_per_tick"] + ) + positions[mask, 1] = g["y_min_mm"] + (py + 0.5) * g["pitch_mm"] + positions[mask, 2] = g["z_min_mm"] + (pz + 0.5) * g["pitch_mm"] + return positions + + @staticmethod + def _particle_labels(event: dict, interactions: dict) -> dict[str, np.ndarray]: + """Build compact truth labels using upstream particle-table rows. + + Parameters + ---------- + event : dict + Prepared image and upstream particle table. + interactions : dict + Upstream interaction truth, including vertices in mm. + + Returns + ------- + dict + Particle sidecar used by fragment and graph supervision. + """ + table, meta = event["columns"], event["meta"] + rows = np.arange(len(table["parent"])) + host = table["fragment_instance"] + valid = (host >= 0) & (host < len(rows)) + interaction = np.full(len(rows), -1, dtype=np.int64) + pid = np.full(len(rows), -1, dtype=np.int64) + primary = np.full(len(rows), -1, dtype=np.int64) + interaction[valid] = table["interaction"][host[valid]] + pid[valid] = [ + PDG_TO_PID.get(abs(int(p)), -1) for p in table["pdg"][host[valid]] + ] + primary[valid] = table["is_primary"][host[valid]] + vertices = np.full((len(rows), 3), np.nan) + if "vertex" in interactions: + known = (interaction >= 0) & (interaction < len(interactions["vertex"])) + vertices[known] = interactions["vertex"][interaction[known]] + return { + "particle": rows, + "group": host.copy(), + "ancestor": table["ancestor"].copy(), + "interaction": interaction, + "nu": np.full(len(rows), -1), + "pid": pid, + "group_primary": np.where(valid, rows == host, -1), + "interaction_primary": primary, + "vertex": (vertices / 10.0 - meta.lower) / meta.size, + "momentum": np.linalg.norm(table["start_mom"][:, :3], axis=1), + "energy_init": table["start_mom"][:, 3].copy(), + "shape": table["motif"].copy(), + } + + def endpoints(self, trees: dict[str, Any]) -> tuple[np.ndarray, np.ndarray]: + """Return stored collected-step endpoints; missing values remain NaN. + + Parameters + ---------- + trees : dict + Nested PIMM event products. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + Start/end positions in mm and times in ns. Non-track ends repeat + their starts. No truth-deposit or creation-point fallback is used. + """ + event = self.prepare(trees) + starts, ends = ( + event["columns"][key].copy() for key in ("first_pos", "last_pos") + ) + ends[event["labels"]["shape"] != 1] = starts[event["labels"]["shape"] != 1] + starts[~np.isfinite(starts)] = np.nan + ends[~np.isfinite(ends)] = np.nan + return starts, ends diff --git a/src/spine/io/parse/supera/cluster.py b/src/spine/io/parse/pimm/cluster.py similarity index 86% rename from src/spine/io/parse/supera/cluster.py rename to src/spine/io/parse/pimm/cluster.py index f4420472..145f0c2a 100644 --- a/src/spine/io/parse/supera/cluster.py +++ b/src/spine/io/parse/pimm/cluster.py @@ -8,12 +8,12 @@ from spine.data import ClusterLabelData -from .base import SuperaParserBase +from .base import PIMMParserBase -__all__ = ["SuperaClusterLabelParser"] +__all__ = ["PIMMClusterLabelParser"] -class SuperaClusterLabelParser(SuperaParserBase): +class PIMMClusterLabelParser(PIMMParserBase): """Return fragment IDs and the winning fragment's particle table row.""" name = "cluster_label" @@ -24,7 +24,7 @@ def __call__(self, trees: dict[str, Any]) -> ClusterLabelData: Parameters ---------- trees : dict[str, Any] - Raw event products returned by SuperaReader. + Raw event products returned by PIMMReader. Returns ------- diff --git a/src/spine/io/parse/supera/metadata.py b/src/spine/io/parse/pimm/metadata.py similarity index 60% rename from src/spine/io/parse/supera/metadata.py rename to src/spine/io/parse/pimm/metadata.py index 1cad058d..55c75174 100644 --- a/src/spine/io/parse/supera/metadata.py +++ b/src/spine/io/parse/pimm/metadata.py @@ -9,12 +9,12 @@ from spine.data import Meta, TensorData from ..base import ParserBase -from .base import SuperaParserBase +from .base import PIMMParserBase -__all__ = ["SuperaMetaParser", "SuperaSourceIDsParser", "SuperaEventInfoParser"] +__all__ = ["PIMMMetaParser", "PIMMEventInfoParser"] -class SuperaMetaParser(SuperaParserBase): +class PIMMMetaParser(PIMMParserBase): """Expose the shared image transform for truth construction in cm.""" name = "meta" @@ -35,31 +35,7 @@ def __call__(self, trees: dict[str, Any]) -> Meta: return self.prepare(trees)["meta"] -class SuperaSourceIDsParser(SuperaParserBase): - """Preserve sparse source IDs separately from internal particle indexes.""" - - name = "source_ids" - - def __call__(self, trees: dict[str, Any]) -> TensorData: - """Return one source identifier per particle-table row. - - Parameters - ---------- - trees : dict[str, Any] - Projected native event products. - - Returns - ------- - TensorData - Integer provenance IDs; batching never shifts these identifiers. - """ - return TensorData( - features=self.prepare(trees)["source_ids"][:, None], - feature_fields={"source_id": (0,)}, - ) - - -class SuperaEventInfoParser(ParserBase): +class PIMMEventInfoParser(ParserBase): """Preserve native event/job identities without fabricating run/subrun IDs.""" name = "event_info" @@ -72,7 +48,7 @@ def __init__(self, dtype: str) -> None: dtype : str Dataset floating-point dtype; identities retain integer precision. """ - super().__init__(dtype, identity_event="events") + super().__init__(dtype, pimm_event="pimm") def __call__(self, trees: dict[str, Any]) -> TensorData: """Return all stored identity fields as a named integer tensor. @@ -87,10 +63,10 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: TensorData One row containing the file's event, input and job identities. """ - fields = sorted(trees["events"]) + fields = sorted(trees["pimm"]["identity"]) return TensorData( features=np.array( - [[trees["events"][key] for key in fields]], dtype=np.int64 + [[trees["pimm"]["identity"][key] for key in fields]], dtype=np.int64 ), feature_fields={key: (column,) for column, key in enumerate(fields)}, ) diff --git a/src/spine/io/parse/supera/particle.py b/src/spine/io/parse/pimm/particle.py similarity index 77% rename from src/spine/io/parse/supera/particle.py rename to src/spine/io/parse/pimm/particle.py index 2f567989..1c1fff19 100644 --- a/src/spine/io/parse/supera/particle.py +++ b/src/spine/io/parse/pimm/particle.py @@ -8,17 +8,17 @@ from spine.data import EdgeIndexData, Meta, ObjectListData, Particle, TensorData -from .base import SuperaParserBase, _shape +from .base import PIMMParserBase, _shape __all__ = [ - "SuperaParticleParser", - "SuperaParticlePointsParser", - "SuperaParticleCoordsParser", - "SuperaParticleGraphParser", + "PIMMParticleParser", + "PIMMParticlePointsParser", + "PIMMParticleCoordsParser", + "PIMMParticleGraphParser", ] -class SuperaParticleParser(SuperaParserBase): +class PIMMParticleParser(PIMMParserBase): """Translate the full stored particle table, preserving event-local IDs.""" name = "particle" @@ -29,7 +29,7 @@ def __call__(self, trees: dict[str, Any]) -> ObjectListData: Parameters ---------- trees : dict[str, Any] - Raw event products returned by SuperaReader. + Raw event products returned by PIMMReader. Returns ------- @@ -41,24 +41,21 @@ def __call__(self, trees: dict[str, Any]) -> ObjectListData: table = event["columns"] particles = [] - # Charge is unavailable here. Integrate truth dE before overlap merging. - event_counts = np.bincount(event["fragment"], minlength=len(table["id"])) - energy_prefix = np.r_[ - 0.0, - np.cumsum( - trees["points"][:, trees["metadata"]["columns"].index("dE")], - dtype=np.float64, - ), - ] + # Hit charge is not a deposited-energy measurement. PIMM's label table + # does not provide per-fragment deposited energy; leave its NaN sentinel. + fragments = event["fragment"] + event_counts = np.bincount( + fragments[fragments >= 0], minlength=len(table["parent"]) + ) starts, ends = self.endpoints(trees) # Preserve the full table so PPN and cluster references remain valid. - for row in range(len(table["id"])): + for row in range(len(table["parent"])): particle = Particle( id=row, - parent_id=int(table["parent_id"][row]), - group_id=int(table["frag_inst_id"][row]), + parent_id=int(table["parent"][row]), + group_id=int(table["fragment_instance"][row]), interaction_id=int(event["labels"]["interaction"][row]), track_id=int(table["geant4_trackid"][row]), pdg_code=int(table["pdg"][row]), @@ -71,7 +68,7 @@ def __call__(self, trees: dict[str, Any]) -> ObjectListData: interaction_primary=int(event["labels"]["interaction_primary"][row]), ancestor_position=event["labels"]["vertex"][row].astype(self.ftype), children_id=np.flatnonzero( - (table["parent_id"] == row) & (table["id"] != row) + (table["parent"] == row) & (np.arange(len(table["parent"])) != row) ).astype(np.int32), end_momentum=table["end_mom"][row, :3].copy(), units="px", @@ -100,15 +97,6 @@ def __call__(self, trees: dict[str, Any]) -> ObjectListData: ) self._fill_ancestry(particle, row, table, event["meta"]) particle.num_voxels = int(event_counts[row]) - # Own-particle ranges need not cover deposits absorbed from omitted - # source particles. Fragment ranges do cover all deposits; assign - # each fragment's energy once, to its stored representative. - start, stop = table["frag_pc_start"][row], table["frag_pc_end"][row] - particle.energy_deposit = 0.0 - if table["frag_id"][row] == row and start >= 0: - particle.energy_deposit = float( - energy_prefix[stop] - energy_prefix[start] - ) particles.append(particle) return ObjectListData(particles, Particle()) @@ -130,7 +118,7 @@ def _fill_ancestry( """ # Missing links must never index the last particle through -1. for relation in ("parent", "ancestor"): - relative = int(table[relation + "_id"][row]) + relative = int(table[relation][row]) if relative >= 0: setattr( particle, @@ -143,7 +131,7 @@ def _fill_ancestry( relation + "_t", float(table["start_pos"][relative, 3]), ) - parent = int(table["parent_id"][row]) + parent = int(table["parent"][row]) if parent >= 0: particle.parent_position = ( (table["start_pos"][parent, :3] / 10.0 - meta.lower) / meta.size @@ -153,7 +141,7 @@ def _fill_ancestry( particle.parent_pdg_code = int(table["geant4_parent_pdg"][row]) -class SuperaParticlePointsParser(SuperaParserBase): +class PIMMParticlePointsParser(PIMMParserBase): """PPN instance starts and track ends from collected-step kinematics.""" name = "particle_points" @@ -164,7 +152,7 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: Parameters ---------- trees : dict[str, Any] - Raw event products returned by SuperaReader. + Raw event products returned by PIMMReader. Returns ------- @@ -177,8 +165,8 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: rows = [] # Only instances represented in the voxel image can provide targets. - for row in np.unique(event["instance"]): - shape = int(_shape(table["inst_sem_type"][row])) + for row in np.unique(event["instance"][event["instance"] >= 0]): + shape = int(_shape(table["instance_sem_type"][row])) if shape not in (0, 1, 2, 3) or abs(table["pdg"][row]) > 1000000000: continue for endpoint, key in enumerate( @@ -201,19 +189,16 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: coordinate_groups={"point": (0, 1, 2)}, feature_fields={"shape": (0,), "particle": (1,), "endpoint": (2,)}, index_cols=np.array([1]), - index_shifts=np.array([len(table["id"])]), + index_shifts=np.array([len(table["parent"])]), ) -class SuperaParticleCoordsParser(SuperaParserBase): +class PIMMParticleCoordsParser(PIMMParserBase): """One pair of collected-step endpoints per stored particle-table row. - Instance representatives use upstream collected-step kinematics. Other - rows use their fragment's earliest/latest deposits. Missing instance - endpoints use its non-LE deposits (all deposits if none are non-LE). - These fallbacks are voxelized deposit positions, not exact step boundaries. - Non-track ends equal their starts. Unavailable coordinates and - times remain NaN; no creation/destruction positions are substituted. + Stored collected-step endpoints are used as provided by PIMM. Missing + coordinates and times remain NaN; no deposit-based approximation or + creation/destruction positions are substituted. Non-track ends repeat starts. """ name = "particle_coords" @@ -230,7 +215,7 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: ------- TensorData Six coordinate columns and two features, with one row per particle. - Point times are converted from microseconds to nanoseconds. + Stored kinematic times remain in nanoseconds. """ event = self.prepare(trees) meta = event["meta"] @@ -252,7 +237,7 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: ) -class SuperaParticleGraphParser(SuperaParserBase): +class PIMMParticleGraphParser(PIMMParserBase): """Directed stored-parent links in the event-local particle namespace. Truth construction contracts invisible rows and groups using this graph. @@ -275,7 +260,8 @@ def __call__(self, trees: dict[str, Any]) -> EdgeIndexData: Directed edges with a span equal to the full particle-table length. """ table = self.prepare(trees)["columns"] - parents, children = table["parent_id"], table["id"] + parents = table["parent"] + children = np.arange(len(parents)) valid = (parents >= 0) & (parents != children) return EdgeIndexData( features=np.stack((parents[valid], children[valid])), span=len(children) diff --git a/src/spine/io/parse/supera/tensor.py b/src/spine/io/parse/pimm/tensor.py similarity index 70% rename from src/spine/io/parse/supera/tensor.py rename to src/spine/io/parse/pimm/tensor.py index 7e83ba23..f925e828 100644 --- a/src/spine/io/parse/supera/tensor.py +++ b/src/spine/io/parse/pimm/tensor.py @@ -6,28 +6,28 @@ from spine.data import TensorData -from .base import SuperaParserBase +from .base import PIMMParserBase -__all__ = ["SuperaTensorParser", "SuperaSegmentationParser"] +__all__ = ["PIMMTensorParser", "PIMMSegmentationParser"] -class SuperaTensorParser(SuperaParserBase): - """Return voxelized truth-deposit energy in MeV.""" +class PIMMTensorParser(PIMMParserBase): + """Return voxelized detector hit charge.""" name = "tensor" def __call__(self, trees: dict[str, Any]) -> TensorData: - """Return summed truth energy in MeV on the shared sparse grid. + """Return summed detector charge on the shared sparse grid. Parameters ---------- trees : dict[str, Any] - Raw event products returned by SuperaReader. + Raw event products returned by PIMMReader. Returns ------- TensorData - Sparse coordinates and summed deposit energy in MeV. + Sparse coordinates and summed charge in the source hit units. """ # Reuse the same grid and overlap winners as the input tensor. event = self.prepare(trees) @@ -38,18 +38,18 @@ def __call__(self, trees: dict[str, Any]) -> TensorData: ) -class SuperaSegmentationParser(SuperaParserBase): +class PIMMSegmentationParser(PIMMParserBase): """Return semantic labels in exactly the input voxel order.""" name = "segmentation" def __call__(self, trees: dict[str, Any]) -> TensorData: - """Return five-class labels; unknown point semantics are rejected. + """Return five-class labels; unlabelled hits use -1. Parameters ---------- trees : dict[str, Any] - Raw event products returned by SuperaReader. + Raw event products returned by PIMMReader. Returns ------- diff --git a/src/spine/io/parse/supera/base.py b/src/spine/io/parse/supera/base.py deleted file mode 100644 index f3488b4f..00000000 --- a/src/spine/io/parse/supera/base.py +++ /dev/null @@ -1,456 +0,0 @@ -"""Aligned voxel, fragment, particle and point-proposal supervision.""" - -from __future__ import annotations - -import json -from typing import Any - -import numpy as np - -from spine.constants import PDG_TO_PID -from spine.data import Meta - -from ..base import ParserBase - - -def _shape(code: int | np.ndarray) -> np.ndarray: - """Translate one-based pysupera classes to SPINE semantic labels. - - Parameters - ---------- - code : int or np.ndarray - Source semantic code or array of codes. - - Returns - ------- - np.ndarray - SPINE labels of the same shape; unknown codes map to ``-1``. - """ - code = np.asarray(code) - return np.array([-1, 0, 1, 3, 2, 4, -1])[np.clip(code, 0, 6)] - - -class SuperaParserBase(ParserBase): - """Voxelize truth deposits once, sharing winner selection across products. - - Coordinates and optional grid arguments are in mm; output metadata uses - cm. The default size comes from the file. The origin defaults to (0,0,0); - an event-local integer translation makes all voxel coordinates nonnegative. - Overlaps sum dE and choose labels by track, shower, delta, Michel, LE, - then earliest time and source row order. Unknown point classes are rejected. - """ - - def __init__( - self, - dtype: str, - voxel_size_mm: float | tuple[float, ...] | None = None, - origin_mm: tuple[float, ...] = (0.0, 0.0, 0.0), - ) -> None: - """Configure a common grid for all parsers in the dataset. - - Parameters - ---------- - dtype : str - Floating-point type used by parser products. - voxel_size_mm : float or sequence[float], optional - Rasterization size; defaults to the file's voxel_size attribute. - origin_mm : sequence[float], default (0, 0, 0) - Common grid origin in detector coordinates (mm). - """ - super().__init__( - dtype, - points_event="points", - particle_event="particles", - metadata_event="metadata", - interaction_event="inter", - ) - - # Normalize grid settings so all parsers can share the event cache. - self.size = ( - None - if voxel_size_mm is None - else tuple(np.broadcast_to(np.asarray(voxel_size_mm, dtype=float), (3,))) - ) - self.origin = tuple(np.asarray(origin_mm, dtype=float)) - if len(self.origin) != 3 or not np.isfinite(self.origin).all(): - raise ValueError("origin_mm must contain three finite values.") - if self.size is not None and ( - not np.isfinite(self.size).all() or min(self.size) <= 0 - ): - raise ValueError("voxel_size_mm must be finite and positive.") - self.grid = (self.size, self.origin) - - def prepare(self, trees: dict[str, Any]) -> dict[str, Any]: - """Build and cache the common voxel representation for all products. - - Parameters - ---------- - trees : dict[str, Any] - Raw event products. A private cache is added to this dictionary. - - Returns - ------- - dict[str, Any] - Unique voxel coordinates, summed energy, winning labels, spatial - metadata and the resolved particle table. - - Raises - ------ - ValueError - If the grid, particle associations or point values are invalid. - """ - cache = trees.setdefault("_supera_prepared", {}) - if self.grid in cache: - return cache[self.grid] - - # Prefer an explicit grid size; otherwise decode the file attribute. - points, table, attrs = trees["points"], trees["particles"], trees["metadata"] - size = self.size - if size is None: - raw = attrs.get("voxel_size") - if raw is None: - raise ValueError( - "Missing voxel_size: configure voxel_size_mm explicitly." - ) - size = json.loads(raw) if isinstance(raw, (str, bytes)) else raw - size = np.broadcast_to(np.asarray(size, dtype=float), (3,)) - if not np.isfinite(size).all() or np.any(size <= 0): - raise ValueError("Invalid voxel size.") - # Resolve sparse source IDs before expanding fragment and instance ranges. - source_ids = table["id"].copy() - table = self._resolve_particles(table) - fragment, instance, semantic = self._point_labels(table, len(points)) - - # Select labels using the same overlap winners as the input tensor. - event = self._voxelize(points, size, attrs, semantic) - winner = event.pop("winner") - event.update( - shape=semantic[winner], - fragment=fragment[winner], - instance=instance[winner], - columns=table, - source_ids=source_ids, - ) - event["labels"] = self._particle_labels(event, trees["inter"]) - cache[self.grid] = event - return event - - @staticmethod - def _particle_labels( - event: dict[str, Any], interactions: dict[str, np.ndarray] - ) -> dict[str, np.ndarray]: - """Build shared particle truth using local IDs and interaction vertices. - - Parameters - ---------- - event : dict[str, Any] - Prepared voxel image and resolved particle columns. - interactions : dict[str, np.ndarray] - Event-local interaction table, with detector vertices in mm. - - Returns - ------- - dict[str, np.ndarray] - Particle labels shared by compact supervision and truth objects. - Missing ancestry yields NaN vertices unless an interaction is known. - """ - table = event["columns"] - host = table["frag_inst_id"] - valid = (host >= 0) & (host < len(table["id"])) - interaction_ids = table["interaction_id"].copy() - interaction_ids[valid] = table["interaction_id"][host[valid]] - vertices = np.full((len(table["id"]), 3), np.nan) - ancestors = table["ancestor_id"] - valid_ancestors = ancestors >= 0 - vertices[valid_ancestors] = table["start_pos"][ancestors[valid_ancestors], :3] - - # A generator interaction may contain several distinct stored roots. - # Use its common vertex instead of arbitrarily selecting one root. - if all(key in interactions for key in ("x", "y", "z")): - positions = np.column_stack([interactions[key] for key in ("x", "y", "z")]) - for row, interaction in enumerate(interactions["interaction_id"]): - if np.isfinite(positions[row]).all(): - vertices[interaction_ids == interaction] = positions[row] - - shapes = _shape(table["sem_type"]) - shapes[valid] = _shape(table["inst_sem_type"][host[valid]]) - shapes[table["frag_sem_type"] == 5] = 4 - - # PID and interaction-primary supervision describe the host instance, - # including its secondary fragments, just as in the LArCV adapter. - pids = np.full(len(table["id"]), -1, dtype=np.int64) - pids[valid] = [ - PDG_TO_PID.get(int(pdg), -1) for pdg in table["pdg"][host[valid]] - ] - primaries = np.full(len(table["id"]), -1, dtype=np.int64) - primaries[valid] = np.where( - table["parent_id"][host[valid]] >= 0, - table["parent_id"][host[valid]] == host[valid], - -1, - ) - - # All relationship fields use the same event-local particle namespace. - return { - "particle": table["id"].copy(), - "group": host.copy(), - "ancestor": table["ancestor_id"].copy(), - "interaction": interaction_ids, - "nu": np.full(len(table["id"]), -1), - "pid": pids, - "group_primary": np.where(valid, table["id"] == host, -1), - "interaction_primary": primaries, - "vertex": (vertices / 10.0 - event["meta"].lower) / event["meta"].size, - "momentum": np.linalg.norm(table["start_mom"][:, :3], axis=1), - "energy_init": table["start_mom"][:, 3].copy(), - "shape": shapes, - } - - def endpoints(self, trees: dict[str, Any]) -> tuple[np.ndarray, np.ndarray]: - """Resolve collected-step endpoints without substituting track creation. - - Parameters - ---------- - trees : dict[str, Any] - Projected native event products. - - Returns - ------- - tuple[np.ndarray, np.ndarray] - Start and end positions in mm, with time in ns in the last column. - Missing endpoints use fragment deposits where available, otherwise - remain NaN. Non-track ends equal their starts. - """ - event = self.prepare(trees) - if "endpoints" in event: - return event["endpoints"] - table = event["columns"] - starts = table["first_pos"].copy() - ends = table["last_pos"].copy() - time_col = next( - i - for i, key in enumerate(trees["metadata"]["columns"]) - if key in ("t", "time") - ) - - # Non-instance rows do not have upstream collected-step kinematics. - for row in range(len(table["id"])): - prefix = "inst" if table["inst_id"][row] == row else "frag" - owner = row if prefix == "inst" else int(table["frag_id"][row]) - if owner >= 0: - lower, upper = ( - table[prefix + "_pc_start"][owner], - table[prefix + "_pc_end"][owner], - ) - # Match upstream's preference for non-LE trajectory deposits. - if prefix == "inst" and table["inst_pc_le_start"][row] > lower: - upper = table["inst_pc_le_start"][row] - points = trees["points"][max(0, lower) : max(0, upper)] - if len(points): - points = points[np.argsort(points[:, time_col], kind="stable")] - for endpoint, point in zip((starts, ends), points[[0, -1]]): - if not np.isfinite(endpoint[row]).all(): - endpoint[row, :3] = point[:3] - endpoint[row, 3] = point[time_col] * 1000.0 - - # A shower has one meaningful collected-step endpoint. - ends[event["labels"]["shape"] != 1] = starts[event["labels"]["shape"] != 1] - starts[~np.isfinite(starts)] = np.nan - ends[~np.isfinite(ends)] = np.nan - event["endpoints"] = (starts, ends) - return starts, ends - - def _voxelize( - self, - points: np.ndarray, - size: np.ndarray, - attrs: dict[str, Any], - semantic: np.ndarray, - ) -> dict[str, Any]: - """Rasterize deposits and retain their overlap winners and transform. - - Parameters - ---------- - points : np.ndarray - Event point rows in detector coordinates. - size : np.ndarray - Three voxel dimensions in mm. - attrs : dict[str, Any] - File attributes, including point column names. - semantic : np.ndarray - SPINE semantic label of each point. - - Returns - ------- - dict[str, Any] - Unique coordinates, summed energy, winning source row and Meta. - """ - num_points = len(points) - # Rasterize on a common grid and translate to nonnegative coordinates. - if not np.isfinite(points[:, :3]).all(): - raise ValueError("Nonfinite point coordinates.") - cells = np.floor((points[:, :3] - self.origin) / size).astype(np.int64) - lower_cell = cells.min(axis=0) if num_points else np.zeros(3, dtype=np.int64) - coords, inverse = np.unique(cells - lower_cell, axis=0, return_inverse=True) - inverse = inverse.reshape(-1) - - energy, winner = self._merge_voxels(points, attrs["columns"], semantic, inverse) - - # Metadata preserves the detector transform in SPINE centimetres. - lower = (np.asarray(self.origin) + lower_cell * size) / 10.0 - count = coords.max(axis=0) + 1 if num_points else np.ones(3, dtype=np.int64) - meta = Meta( - lower=lower, - upper=lower + count * size / 10.0, - size=size / 10.0, - count=count, - ) - return {"coords": coords, "energy": energy, "winner": winner, "meta": meta} - - @staticmethod - def _merge_voxels( - points: np.ndarray, - columns: tuple[str, ...], - semantic: np.ndarray, - inverse: np.ndarray, - ) -> tuple[np.ndarray, np.ndarray]: - """Sum energy and resolve overlap labels by class, time and row order. - - Parameters - ---------- - points : np.ndarray - Event point rows before duplicate removal. - columns : tuple[str, ...] - Physical column names in the point array. - semantic : np.ndarray - SPINE semantic label for each contribution. - inverse : np.ndarray - Unique voxel index of each contribution. - - Returns - ------- - tuple[np.ndarray, np.ndarray] - Summed voxel energies and winning source point indexes. - """ - num_points = len(points) - values = points[:, columns.index("dE")] - times = points[:, columns.index("t" if "t" in columns else "time")] - if not np.isfinite(values).all() or not np.isfinite(times).all(): - raise ValueError("Nonfinite deposit energy or time.") - ranks = np.full(num_points, 5) - for rank, shape in enumerate((1, 0, 3, 2, 4)): - ranks[semantic == shape] = rank - order = np.lexsort((np.arange(num_points), times, ranks, inverse)) - winner = ( - order[np.r_[True, np.diff(inverse[order]) != 0]] if num_points else order - ) - energy = np.bincount(inverse, weights=values, minlength=0) - - return energy, winner - - @staticmethod - def _resolve_particles(table: dict[str, np.ndarray]) -> dict[str, np.ndarray]: - """Resolve sparse particle relationships to event-local table indexes. - - Parameters - ---------- - table : dict[str, np.ndarray] - Raw particle columns with source IDs and rebased point ranges. - - Returns - ------- - dict[str, np.ndarray] - Shallow copy with relationship columns replaced by row indexes. - - Raises - ------ - ValueError - If particle IDs are invalid or a relationship cannot be resolved. - """ - source_ids = table["id"] - num_particles = len(source_ids) - if len(np.unique(source_ids)) != num_particles or np.any(source_ids < 0): - raise ValueError("Supera particle ids must be unique and nonnegative.") - # Contrary to io_v3's index-convention prose, layout.py retains sparse - # source IDs. Resolve relationships explicitly, never index by an ID. - row_of = {int(pid): row for row, pid in enumerate(source_ids)} - table = dict(table) - for key in ( - "id", - "parent_id", - "ancestor_id", - "frag_id", - "inst_id", - "frag_inst_id", - ): - try: - table[key] = np.array( - [row_of[int(v)] if v >= 0 else -1 for v in table[key]], - dtype=np.int64, - ) - except KeyError as err: - raise ValueError( - f"Dangling Supera particle relationship in {key}." - ) from err - return table - - @staticmethod - def _point_labels( - table: dict[str, np.ndarray], num_points: int - ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Expand and validate fragment and instance ownership of every point. - - Parameters - ---------- - table : dict[str, np.ndarray] - Particle columns with event-local relationships and point ranges. - num_points : int - Number of point rows in this event. - - Returns - ------- - tuple[np.ndarray, np.ndarray, np.ndarray] - Fragment row, instance row and SPINE semantic label for each point. - - Raises - ------ - ValueError - If ranges overlap, leave gaps or disagree with stored relationships - or semantic classes. - """ - # Each representative owns one half-open range; no point is shared. - fragment = np.full(num_points, -1, dtype=np.int64) - instance = np.full(num_points, -1, dtype=np.int64) - semantic = np.full(num_points, -1, dtype=np.int64) - for level, dest in (("frag", fragment), ("inst", instance)): - for row in np.flatnonzero(table[level + "_id"] == table["id"]): - start, stop = int(table[level + "_pc_start"][row]), int( - table[level + "_pc_end"][row] - ) - if start == stop == -1: - continue - if not 0 <= start <= stop <= num_points or np.any( - dest[start:stop] >= 0 - ): - raise ValueError(f"Invalid or overlapping {level} point ranges.") - dest[start:stop] = row - if level == "inst": - split = int(table["inst_pc_le_start"][row]) - if not start <= split <= stop: - raise ValueError("Invalid instance LE range.") - semantic[start:split] = _shape(table["inst_sem_type"][row]) - semantic[split:stop] = 4 - - # Cross-check both partitions and the explicit fragment-to-instance join. - if np.any(fragment < 0) or np.any(instance < 0): - raise ValueError("Fragment and instance ranges must cover every point.") - if np.any(semantic < 0): - raise ValueError( - "Unknown Supera semantic class cannot supervise a five-class model." - ) - if num_points and not np.array_equal(table["frag_inst_id"][fragment], instance): - raise ValueError("Fragment ownership disagrees with instance ranges.") - if num_points and not np.array_equal( - table["frag_sem_type"][fragment] == 5, semantic == 4 - ): - raise ValueError("Fragment semantics disagree with instance LE ranges.") - return fragment, instance, semantic diff --git a/src/spine/io/read/__init__.py b/src/spine/io/read/__init__.py index 6cd56b71..c640f3b0 100644 --- a/src/spine/io/read/__init__.py +++ b/src/spine/io/read/__init__.py @@ -3,6 +3,6 @@ from .cache import CacheReader from .hdf5 import HDF5Reader from .larcv import LArCVReader -from .supera import SuperaReader +from .pimm import PIMMReader -__all__ = ["CacheReader", "HDF5Reader", "LArCVReader", "SuperaReader"] +__all__ = ["CacheReader", "HDF5Reader", "LArCVReader", "PIMMReader"] diff --git a/src/spine/io/read/pimm.py b/src/spine/io/read/pimm.py new file mode 100644 index 00000000..aaa77836 --- /dev/null +++ b/src/spine/io/read/pimm.py @@ -0,0 +1,159 @@ +"""Event access to aligned PIMM detector hits and label tables.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from importlib import import_module +from typing import Any, ClassVar + +import h5py +import numpy as np + +from .base import ReaderBase + +__all__ = ["PIMMReader"] + + +class PIMMReader(ReaderBase): + """Delegate detector decoding and truth associations to ``pimm-data``. + + The upstream dataset joins hits and ``labl`` by task and event identity. + SPINE reads its untransformed NumPy products, preserving particle-table + row indexes until SPINE collation. No legacy pysupera file is read here. + """ + + name: ClassVar[str] = "pimm" + + # Public discovery options mirror the upstream dataset API. + # pylint: disable=too-many-arguments,too-many-positional-arguments + def __init__( + self, + data_root: str = ".", + files: Mapping[str, str | list[str]] | None = None, + runs: str | list[str] | None = None, + split: str | None = None, + dataset_name: str = "sim_pixel", + volume: int | None = None, + keys: Sequence[str] | None = None, + **selection: Any, + ) -> None: + """Index paired detector-hit and label shards. + + Parameters + ---------- + data_root : str, default "." + PIMM corpus root, containing modality or task directories. + files : mapping, optional + Explicit ``hits`` and ``labl`` paths or glob patterns. + runs : str or list[str], optional + Task directories or glob relative to the corpus root. + split : str, optional + Upstream split directory; defaults to flat discovery without runs. + dataset_name : str, default "sim_pixel" + Upstream file prefix. + volume : int, optional + Restrict input to one detector volume. + keys : sequence[str], optional + Only ``pimm`` is supported: it is one aligned upstream event. + **selection : dict + Standard ReaderBase entry selection. No run/subrun is invented. + """ + if keys is not None and set(keys) != {"pimm"}: + raise ValueError("PIMMReader exposes the aligned 'pimm' event product.") + try: + backend = import_module("pimm_data") + except ImportError as err: + raise ImportError( + "PIMM input requires the optional spine[pimm] extra." + ) from err + + # Keep all ownership references, including LE; semantic losses decide + # which classes to supervise. Never apply upstream transforms twice. + self.dataset = backend.JAXTPCDataset( + data_root=data_root, + files=files, + runs=runs, + split=("train" if runs is not None else "") if split is None else split, + dataset_name=dataset_name, + modalities=("hits",), + labels="all", + le_is_stuff=False, + volume=volume, + allow_partial=False, + ) + reader = self.dataset.hits_reader + self.file_paths = list(reader.h5_files) + counts = np.array([len(index) for index in reader.indices]) + self.num_entries = int(counts.sum()) + self.file_offsets = np.r_[0, np.cumsum(counts)[:-1]].astype(np.int64) + self.file_index = np.repeat(np.arange(len(counts)), counts) + self.process_entry_list(**selection) + + self.metadata = [ + self._read_metadata(path) for path in self.dataset.labl_reader.h5_files + ] + + @staticmethod + def _read_metadata(path: str) -> dict[str, Any]: + """Read the small detector-geometry attributes from a label shard. + + Parameters + ---------- + path : str + PIMM labl shard path. + + Returns + ------- + dict + Pixel geometry and coordinate/value conventions. + """ + with h5py.File(path, "r") as file: + attrs: dict[str, Any] = dict(file["config"].attrs) + geometry = attrs.get("supera_pixel_geometry") + return { + "pixel_geometry": json.loads(geometry) if geometry is not None else None, + "hit_x_from": attrs.get("supera_hit_x_from"), + "hit_energy": attrs.get("supera_hit_energy"), + } + + def get(self, idx: int) -> dict[str, Any]: + """Read one aligned event and its physical source identity. + + Parameters + ---------- + idx : int + Index in SPINE's selected entry list. + + Returns + ------- + dict + Nested ``pimm`` event and standard source provenance. + """ + entry = int(self.entry_index[idx]) + file_index, event_num = self.dataset.hits_reader.locate(entry) + labl_index, _ = self.dataset.labl_reader.locate(entry) + event = self.dataset.get_data(entry) + event["metadata"] = self.metadata[labl_index] + identity = self.dataset.hits_reader.event_id_of(entry) + event["identity"] = {"entry": event_num} + if identity is not None: + event["identity"]["event_id"] = identity + result = { + "index": idx, + "file_index": file_index, + "file_entry_index": event_num, + "pimm": event, + } + result.update(self.get_source_provenance(file_index, event_num)) + return result + + def close(self) -> None: + """Release upstream HDF5 handles before leaving a worker.""" + self.dataset.hits_reader.close() + self.dataset.labl_reader.close() + + def __getstate__(self) -> dict[str, Any]: + """Drop live handles before pickling for spawned loader workers.""" + self.close() + return self.__dict__.copy() diff --git a/src/spine/io/read/supera.py b/src/spine/io/read/supera.py deleted file mode 100644 index b3270150..00000000 --- a/src/spine/io/read/supera.py +++ /dev/null @@ -1,267 +0,0 @@ -"""Projected event reads from the pysupera 4.1 HDF5 format.""" - -from __future__ import annotations - -from collections.abc import Iterable, Sequence -from importlib import import_module -from typing import Any, ClassVar, cast - -import h5py -import numpy as np - -from .base import ReaderBase -from .hdf5.common import require_group - -__all__ = ["SuperaReader"] - - -def _dataset(file: h5py.File | h5py.Group, key: str) -> h5py.Dataset: - """Retrieve a required dataset and reject a group at the same path. - - Parameters - ---------- - file : h5py.File or h5py.Group - Open input file or parent group. - key : str - Required dataset path. - - Returns - ------- - h5py.Dataset - Dataset handle owned by the open file. - - Raises - ------ - TypeError - If the path refers to a group rather than a dataset. - """ - value = file[key] - if not isinstance(value, h5py.Dataset): - raise TypeError(f"Expected a dataset at {key}.") - return value - - -class SuperaReader(ReaderBase): - """Read truth deposits and event-local particle ranges without loading hits. - - Products are ``points``, ``particles``, ``inter``, ``events``, ``metadata`` - and ``groups``. Hit labels require an explicit plane product such as - ``hit_labels/volume0/plane0``. No HDF5 handles survive a read, making the - reader safe to pickle or use in data-loader workers. - """ - - name: ClassVar[str] = "supera" - default_keys = ("points", "particles", "inter", "events", "metadata") - - def __init__( - self, - file_keys: str | list[str] | None = None, - file_list: str | None = None, - keys: Sequence[str] | None = None, - limit_num_files: int | None = None, - max_print_files: int = 10, - **selection: Any, - ) -> None: - """Resolve files, validate format and apply ReaderBase entry selection. - - Parameters - ---------- - file_keys : str or list[str], optional - Input paths or glob patterns. - file_list : str, optional - Text file containing input paths. - keys : sequence[str], optional - Raw products to read; defaults exclude groups and hit labels. - limit_num_files : int, optional - Maximum number of input files. - max_print_files : int, default 10 - Maximum number of paths printed while resolving files. - **selection : dict - Entry selection arguments accepted by ReaderBase.process_entry_list. - Run/event selection is unavailable because pysupera does not define - run/subrun identities. - """ - # Register optional compression filters used by upstream productions. - try: - import_module("hdf5plugin") - except ImportError: - pass - - # Inspect only format metadata and offsets while building the event index. - self.process_file_paths(file_keys, file_list, limit_num_files, max_print_files) - self.keys = tuple(self.default_keys if keys is None else keys) - counts = [self._validate_file(path) for path in self.file_paths] - - # ReaderBase maps selected global entries back to their physical files. - self.num_entries = sum(counts) - self.file_offsets = np.r_[0, np.cumsum(counts)[:-1]].astype(np.int64) - self.file_index = np.repeat(np.arange(len(counts)), counts) - self.process_entry_list(**selection) - - def _validate_file(self, path: str) -> int: - """Check format metadata without materializing event payloads. - - Parameters - ---------- - path : str - Input HDF5 path. - - Returns - ------- - int - Number of events in the validated file. - - Raises - ------ - TypeError - If the format, offsets or required HDF5 object types are invalid. - ValueError - If the point layout or requested products are invalid. - """ - # Pylint infers Group for h5py lookups even after _dataset validates them. - # pylint: disable=no-member - with h5py.File(path, "r") as file: - if ( - "format_version" not in file - or _dataset(file, "format_version").asstr()[()] != "4.1.0" - ): - raise TypeError(f"{path}: expected pysupera format 4.1.0.") - - # Each table has its own event offsets and row domain. - num_entries = int(_dataset(file, "n_events")[()]) - for offset, target in ( - ("events/offsets", "particles/id"), - ("points/offsets", "points/flat"), - ("inter/offsets", "inter/interaction_id"), - ): - offsets = _dataset(file, offset)[:] - if ( - offsets.shape != (num_entries + 1,) - or offsets[0] != 0 - or np.any(np.diff(offsets) < 0) - or offsets[-1] != len(_dataset(file, target)) - ): - raise TypeError(f"{path}: invalid {offset}.") - - # Confirm column names and physical width before parsing point data. - columns = tuple(_dataset(file, "points/columns").asstr()[:]) - if ( - columns[:3] != ("x", "y", "z") - or "dE" not in columns - or not {"t", "time"}.intersection(columns) - or _dataset(file, "points/flat").ndim != 2 - or _dataset(file, "points/flat").shape[1] != len(columns) - ): - raise ValueError(f"{path}: unsupported point columns {columns}.") - - # Large hit-label arrays are opt-in through an explicit plane product. - for key in self.keys: - if key not in self.default_keys + ("groups",) and not ( - key.startswith("hit_labels/") - and key.count("/") == 2 - and key in file - and isinstance(file[key], h5py.Group) - and "offsets" in require_group(file, key) - ): - raise ValueError(f"Unsupported Supera product: {key}") - return num_entries - - def get(self, idx: int) -> dict[str, Any]: - """Read selected products for one event, rebasing absolute ranges. - - Parameters - ---------- - idx : int - Index in the selected entry list. - - Returns - ------- - dict[str, Any] - Projected products and standard source-file provenance. Point and - particle ranges are local to the returned event. - """ - # Dataset members are validated by _dataset; h5py lookup inference is dynamic. - # pylint: disable=no-member - file_index = self.get_file_index(idx) - entry_index = self.get_file_entry_index(idx) - result: dict[str, Any] = { - "index": idx, - "file_index": file_index, - "file_entry_index": entry_index, - } - result.update(self.get_source_provenance(file_index, entry_index)) - - # Open per read so worker processes never inherit live HDF5 handles. - with h5py.File(self.file_paths[file_index], "r") as file: - point_start, point_stop = _dataset(file, "points/offsets")[ - entry_index : entry_index + 2 - ] - - # Hit labels require an explicit plane path; never default to all hits. - for key in self.keys: - if key == "metadata": - result[key] = dict(file.attrs) - result[key]["columns"] = tuple( - _dataset(file, "points/columns").asstr()[:] - ) - elif key == "points": - result[key] = _dataset(file, "points/flat")[point_start:point_stop] - elif key == "events": - result[key] = { - name: _dataset(file, f"{key}/{name}")[entry_index] - for name in require_group(file, key) - if name != "offsets" - } - else: - # Each physical table has its own event offsets. - result[key] = self._read_table(file, key, entry_index) - return result - - @staticmethod - def _read_table( - file: h5py.File, key: str, entry_index: int - ) -> dict[str, np.ndarray]: - """Slice a columnar product and rebase its stored ranges. - - Parameters - ---------- - file : h5py.File - Open input file. - key : str - Particle, interaction, group or explicit hit-plane product path. - entry_index : int - Event index within this file. - - Returns - ------- - dict[str, np.ndarray] - Event-local columns. Hit and deposit-group labels retain their own - row domains and are never assumed to align with points. - """ - group = require_group(file, key) - offset_key = "events/offsets" if key == "particles" else f"{key}/offsets" - start, stop = _dataset(file, offset_key)[entry_index : entry_index + 2] - point_start = _dataset(file, "points/offsets")[entry_index] - particle_start = _dataset(file, "events/offsets")[entry_index] - - # Slice each column before performing any range arithmetic. - table: dict[str, np.ndarray] = {} - # HDF5 groups yield child names; their iterator typing is overly broad. - for name in cast(Iterable[str], group): - if name in ("offsets", "volume_offsets", "volume_offsets_index"): - continue - values = _dataset(group, name)[start:stop] - if key in ("particles", "inter"): - if "pc_" in name: - values = np.where(values >= 0, values - point_start, values) - elif key == "inter" and name in ("part_start", "part_end"): - values = np.where(values >= 0, values - particle_start, values) - table[name] = values - - # Volume starts index the event's deposit-group array, not its points. - if key == "groups": - start, stop = _dataset(group, "volume_offsets_index")[ - entry_index : entry_index + 2 - ] - table["volume_offsets"] = _dataset(group, "volume_offsets")[start:stop] - return table diff --git a/test/test_io/test_supera/__init__.py b/test/test_io/test_pimm/__init__.py similarity index 100% rename from test/test_io/test_supera/__init__.py rename to test/test_io/test_pimm/__init__.py diff --git a/test/test_io/test_pimm/conftest.py b/test/test_io/test_pimm/conftest.py new file mode 100644 index 00000000..756ab39a --- /dev/null +++ b/test/test_io/test_pimm/conftest.py @@ -0,0 +1,127 @@ +"""Small native PIMM events used to check SPINE adaptation.""" + +import numpy as np +import pytest + +FULL_SCHEMA = { + key: {"provider": provider} + for key, provider in ( + ("data", "tensor"), + ("seg_label", "segmentation"), + ("clust_label", "cluster_label"), + ("particles", "particle"), + ("coord_label", "particle_coords"), + ("ppn_label", "particle_points"), + ("meta", "meta"), + ("graph_label", "particle_graph"), + ("event_info", "event_info"), + ) +} + + +@pytest.fixture +def raw(): + """Pixel charge contributions with overlap, LE, an unknown hit and ancestry.""" + table = dict( + parent=np.array([0, 0, 0]), + ancestor=np.array([0, 0, 0]), + fragment=np.arange(3), + instance=np.array([0, 1, -1]), + fragment_instance=np.array([0, 1, 0]), + interaction=np.array([0, 0, 1]), + pdg=np.array([13, 22, 11]), + parent_pdg=np.array([13, 13, 13]), + geant4_trackid=np.array([10, 80, 150]), + geant4_parent_trackid=np.array([-1, 10, 10]), + geant4_parent_pdg=np.array([0, 13, 13]), + motif=np.array([1, 0, 4]), + instance_sem_type=np.array([2, 1, -1]), + is_primary=np.array([1, 0, 0]), + ) + for key in ("start_pos", "end_pos", "first_pos", "last_pos"): + table[key] = np.array( + [[0.5, 0.5, 0.5, 10.0], [1.5, 0.5, 0.5, 20.0], [2.5, 0.5, 0.5, 30.0]] + ) + table["last_pos"][0, 0] = 2.5 + for key in ("start_mom", "end_mom"): + table[key] = np.tile([0.0, 0.0, 100.0, 150.0], (3, 1)) + geometry = dict( + x_anode_mm=0.0, + drift_direction=-1, + reference_tick=0.0, + mm_per_tick=1.0, + y_min_mm=0.0, + z_min_mm=0.0, + pitch_mm=1.0, + ) + return { + "pimm": dict( + hits=dict( + coord=np.array( + [ + [0.0, 0.0, 0.5], + [0.0, 0.0, 1.5], + [0.0, 0.0, 2.5], + [0.0, 0.0, 0.5], + [0.0, 0.0, 3.5], + ] + ), + energy=np.array([[2.0], [3.0], [4.0], [9.0], [-1.0]]), + plane_id=np.zeros((5, 1), int), + planes=["volume_0_Pixel"], + readout_type="pixel", + segment_motif=np.array([[1], [0], [4], [0], [-1]]), + instance_fragment=np.array([[0], [1], [2], [1], [-1]]), + instance_particle=np.array([[0], [1], [0], [1], [-1]]), + ), + particle=table, + interaction={"vertex": np.array([[10.0, 20.0, 30.0], [99.0, 99.0, 99.0]])}, + metadata=dict( + pixel_geometry=[geometry], hit_x_from="nominal", hit_energy="charge" + ), + identity={"event_id": 12345, "entry": 0}, + ) + } + + +def round_trip_cache(batch, products, path, stage): + """Publish native products and reload them through cache collation. + + Parameters + ---------- + batch : dict + Collated input batch containing physical-source provenance. + products : dict + Batched products to cache, without source metadata. + path : pathlib.Path + Temporary cache repository. + stage : str + Cache stage name. + + Returns + ------- + dict + Re-collated products from the published cache stage. + """ + from spine.io.collate import CollateAll + from spine.io.read import CacheReader + from spine.io.unwrap import Unwrapper + from spine.io.write import CacheWriter + + metadata = { + key: value + for key, value in batch.items() + if key == "index" or key.startswith("source_file_") + } + payload = { + key: value.to_numpy() if hasattr(value, "to_numpy") else value + for key, value in products.items() + } + writer = CacheWriter(path=str(path), stage=stage, keys=list(products)) + writer(Unwrapper()({**metadata, **payload}), {}) + writer.finalize() + writer.close() + reader = CacheReader(path=str(path), stage=stage) + events = reader.get_many(list(range(len(batch["index"])))) + reader.close() + return CollateAll(data_keys=products)(events) diff --git a/test/test_io/test_pimm/coverage.ini b/test/test_io/test_pimm/coverage.ini new file mode 100644 index 00000000..c36b93cf --- /dev/null +++ b/test/test_io/test_pimm/coverage.ini @@ -0,0 +1,13 @@ +# Strict coverage for the PIMM adapter, including error paths. +[run] +branch = True +source = src/spine + +[report] +include = + src/spine/io/read/pimm.py + src/spine/io/dataset/pimm.py + src/spine/io/parse/pimm/* +exclude_lines = +fail_under = 100 +show_missing = True diff --git a/test/test_io/test_pimm/test_parsers.py b/test/test_io/test_pimm/test_parsers.py new file mode 100644 index 00000000..4f9947e0 --- /dev/null +++ b/test/test_io/test_pimm/test_parsers.py @@ -0,0 +1,211 @@ +"""PIMM charge/label alignment, truth and malformed-input regression tests.""" + +import copy + +import numpy as np +import pytest + +from spine.io.parse.pimm import ( + PIMMClusterLabelParser, + PIMMEventInfoParser, + PIMMMetaParser, + PIMMParticleCoordsParser, + PIMMParticleGraphParser, + PIMMParticleParser, + PIMMParticlePointsParser, + PIMMSegmentationParser, + PIMMTensorParser, +) + + +def test_aligned_charge_and_truth(raw): + """Charge overlaps share winners; unknown inputs keep an ignored target.""" + parser = PIMMTensorParser("float32", voxel_size_mm=1, include_unlabeled=True) + data = parser(raw) + seg = PIMMSegmentationParser("float32", voxel_size_mm=1, include_unlabeled=True)( + raw + ) + clust = PIMMClusterLabelParser("float32", voxel_size_mm=1, include_unlabeled=True)( + raw + ) + np.testing.assert_array_equal(data.features[:, 0], [11, 3, 4, -1]) + np.testing.assert_array_equal(seg.features[:, 0], [1, 0, 4, -1]) + np.testing.assert_array_equal(data.coords, clust.coords) + np.testing.assert_array_equal(data.coords, seg.coords) + np.testing.assert_array_equal(clust.particles["particle"], [0, 1, 2]) + np.testing.assert_array_equal(clust.particles["interaction"], [0, 0, 0]) + np.testing.assert_array_equal(clust.particles["pid"], [2, 0, 2]) + assert ( + PIMMMetaParser("float32", voxel_size_mm=1, include_unlabeled=True)(raw) + is data.meta + ) + assert len(raw["_pimm_prepared"]) == 1 + particles = PIMMParticleParser("float32", voxel_size_mm=1, include_unlabeled=True)( + raw + ) + assert all(np.isnan(p.energy_deposit) for p in particles) + assert particles[0].parent_track_id == -1 + assert particles[1].pid == 0 + assert particles[0].interaction_primary == 1 + np.testing.assert_array_equal(particles[0].children_id, [1, 2]) + np.testing.assert_array_equal( + PIMMParticleGraphParser("float32")(raw).index, [[0, 0], [1, 2]] + ) + identity = PIMMEventInfoParser("float32")(raw) + assert identity.features.shape == (1, 2) + assert 12345 in identity.features + + +def test_endpoints_and_missing_ancestry(raw): + """Missing endpoints remain unknown; -1 never indexes the final row.""" + table = raw["pimm"]["particle"] + table["first_pos"][2] = np.nan + table["ancestor"][2] = table["parent"][2] = -1 + table["fragment_instance"][2] = -1 + coords = PIMMParticleCoordsParser("float32", voxel_size_mm=1)(raw) + assert np.isnan(coords.coordinate_data[2]).all() + particles = PIMMParticleParser("float32", voxel_size_mm=1)(raw) + assert particles[2].ancestor_track_id == -1 + ppn = PIMMParticlePointsParser("float32", voxel_size_mm=1)(raw) + assert len(ppn.coords) == 3 + assert ppn.features.shape[1] == 3 + assert set(coords.coordinate_groups) == {"start", "end"} + + +def test_empty_event(raw): + """Empty point clouds and tables retain stable dimensions.""" + event = raw["pimm"] + for key, value in event["hits"].items(): + if isinstance(value, np.ndarray): + event["hits"][key] = value[:0] + for key, value in event["particle"].items(): + event["particle"][key] = value[:0] + for parser in ( + PIMMTensorParser, + PIMMSegmentationParser, + PIMMClusterLabelParser, + PIMMParticleCoordsParser, + PIMMParticlePointsParser, + ): + assert len(parser("float32")(raw).features) == 0 + assert len(PIMMParticleParser("float32")(raw)) == 0 + assert PIMMParticleGraphParser("float32")(raw).index.shape == (2, 0) + + +@pytest.mark.parametrize( + "kwargs", + [ + dict(voxel_size_mm=0), + dict(voxel_size_mm=np.nan), + dict(origin_mm=(0, 0)), + dict(origin_mm=(0, np.inf, 0)), + ], +) +def test_invalid_grid(kwargs): + """Reject grids that cannot define a detector-to-voxel transform.""" + with pytest.raises(ValueError): + PIMMTensorParser("float32", **kwargs) + + +@pytest.mark.parametrize( + "part,key,value", + [ + ("hits", "readout_type", "wire"), + ("metadata", "pixel_geometry", None), + ("metadata", "hit_x_from", "true_t0"), + ("metadata", "hit_energy", "energy"), + ("hits", "energy", np.array([[np.nan]] * 5)), + ("hits", "instance_fragment", np.array([[99]] * 5)), + ("hits", "segment_motif", np.array([[8]] * 5)), + ("hits", "energy", np.array([[1.0]])), + ], +) +def test_invalid_event(raw, part, key, value): + """Reject unsupported geometry, corrupt references and misaligned arrays.""" + raw["pimm"][part][key] = value + with pytest.raises(ValueError): + PIMMTensorParser("float32")(raw) + + +def test_batch_and_truth_round_trip(raw, tmp_path): + """Particle references shift per event and physical truth survives HDF5.""" + from spine.config.factory import instantiate + from spine.construct import BuildManager + from spine.io.collate import CollateAll + from spine.io.dataset import PIMMDataset + from spine.io.read import HDF5Reader + from spine.io.unwrap import Unwrapper + from spine.io.write import HDF5Writer + + from .conftest import FULL_SCHEMA + + events = [] + for index in range(2): + trees = copy.deepcopy(raw) + event = { + key: instantiate( + PIMMDataset.parser_dict, + dict(cfg, voxel_size_mm=1) if key != "event_info" else cfg, + dtype="float32", + )(trees) + for key, cfg in FULL_SCHEMA.items() + } + # Unlabelled hits do not contribute to truth objects. + event["index"] = index + events.append(event) + batch = CollateAll(data_keys=events[0])(events) + restored = Unwrapper()(batch) + BuildManager(True, True, True, mode="truth", units="cm")(restored) + assert len(restored["truth_particles"][0]) > 0 + assert restored["truth_particles"][0][0].units == "cm" + path = tmp_path / "truth.h5" + writer = HDF5Writer( + file_name=str(path), + keys=list(FULL_SCHEMA) + + ["truth_fragments", "truth_particles", "truth_interactions"], + format_version=2, + ) + writer(restored, {}) + writer.finalize() + writer.close() + reloaded = HDF5Reader(file_keys=str(path))[0] + assert len(reloaded["truth_particles"]) == len(restored["truth_particles"][0]) + + +def test_unavailable_truth_and_ppn_rejection(raw): + """Unknown vertices and unsupported or missing endpoints do not become targets.""" + event = raw["pimm"] + event["interaction"] = {} + table = event["particle"] + table.pop("geant4_parent_trackid") + table["instance_sem_type"][0] = 5 + table["first_pos"][1] = np.nan + assert len(PIMMParticlePointsParser("float32")(raw).features) == 0 + assert all( + np.isnan(p.ancestor_position).all() for p in PIMMParticleParser("float32")(raw) + ) + + +def test_opposite_drift_volumes(raw): + """Different anode faces map identical ticks to opposite physical positions.""" + geometry = raw["pimm"]["metadata"]["pixel_geometry"] + geometry.append( + dict(geometry[0], x_anode_mm=10.0, drift_direction=1, reference_tick=1.0) + ) + hits = raw["pimm"]["hits"] + hits["planes"].append("volume_1_Pixel") + hits["plane_id"][1] = 1 + positions = PIMMTensorParser._coordinates(hits, raw["pimm"]["metadata"]) + np.testing.assert_allclose(positions[0], [0.5, 0.5, 0.5]) + np.testing.assert_allclose(positions[1], [9.5, 0.5, 0.5]) + + +def test_unlabelled_voxels_excluded_for_training(raw): + """Default training retains charge from all contributors in labelled voxels.""" + tensor = PIMMTensorParser("float32", voxel_size_mm=1)(raw) + labels = PIMMSegmentationParser("float32", voxel_size_mm=1)(raw) + np.testing.assert_array_equal(tensor.features[:, 0], [11, 3, 4]) + assert np.all(labels.features >= 0) + raw["pimm"]["hits"]["segment_motif"][:] = -1 + raw.pop("_pimm_prepared") + assert len(PIMMTensorParser("float32")(raw).features) == 0 diff --git a/test/test_io/test_supera/test_supera_chain.py b/test/test_io/test_pimm/test_pimm_chain.py similarity index 81% rename from test/test_io/test_supera/test_supera_chain.py rename to test/test_io/test_pimm/test_pimm_chain.py index 5c3cbd36..736058c4 100644 --- a/test/test_io/test_supera/test_supera_chain.py +++ b/test/test_io/test_pimm/test_pimm_chain.py @@ -8,38 +8,28 @@ import pytest from spine.io.collate import CollateAll -from spine.io.dataset import SuperaDataset +from spine.io.dataset import PIMMDataset from .conftest import FULL_SCHEMA, round_trip_cache -from .test_supera_truth import check_truth_round_trip +from .truth_helpers import check_truth_round_trip -SAMPLE = os.environ.get("SPINE_SUPERA_SAMPLE") -pytestmark = pytest.mark.skipif(not SAMPLE, reason="Set SPINE_SUPERA_SAMPLE") +SAMPLE = os.environ.get("SPINE_PIMM_ROOT") +pytestmark = pytest.mark.skipif(not SAMPLE, reason="Set SPINE_PIMM_ROOT") def test_real_truth_conversion(tmp_path): """Build two complete real events and preserve cm geometry and associations.""" - ds = SuperaDataset(file_keys=SAMPLE, schema=FULL_SCHEMA, dtype="float32", n_entry=2) + ds = PIMMDataset( + data_root=SAMPLE, + dataset_name="sim_pixel", + schema=FULL_SCHEMA, + dtype="float32", + n_entry=2, + ) events = [ds[0], ds[1]] truth = check_truth_round_trip(events, tmp_path / "truth.h5") for index, event in enumerate(events): - raw = ds.reader[index] - interactions = raw["inter"] - for interaction in truth["truth_interactions"][index]: - row = np.flatnonzero(interactions["interaction_id"] == interaction.orig_id)[ - 0 - ] - expected = ( - np.array([interactions[key][row] for key in ("x", "y", "z")]) / 10 - ) - np.testing.assert_allclose(interaction.vertex, expected, atol=1e-4) - # Fragment-attributed energy counts every source deposit exactly once. - energy_col = raw["metadata"]["columns"].index("dE") - assert np.isclose( - sum(p.energy_deposit for p in event["particles"]), - raw["points"][:, energy_col].sum(), - rtol=1e-5, - ) + assert all(np.isnan(p.energy_deposit) for p in event["particles"]) assert len(truth["truth_fragments"][index]) > 0 assert len(truth["truth_particles"][index]) > 0 assert any(len(p.children_id) for p in truth["truth_particles"][index]) @@ -55,7 +45,13 @@ def test_real_grappa_targets_and_backward(path, tmp_path): from spine.model.grappa import GrapPA, GrapPALoss torch.manual_seed(7) - ds = SuperaDataset(file_keys=SAMPLE, schema=FULL_SCHEMA, dtype="float32", n_entry=2) + ds = PIMMDataset( + data_root=SAMPLE, + dataset_name="sim_pixel", + schema=FULL_SCHEMA, + dtype="float32", + n_entry=2, + ) # Two events exercise the coordinate-table offsets used by orientation targets. batch = CollateAll(data_keys=ds.data_keys)([ds[0], ds[1]]) labels = batch["clust_label"].to_tensor(dtype=torch.float32) diff --git a/test/test_io/test_supera/test_supera_model.py b/test/test_io/test_pimm/test_pimm_model.py similarity index 92% rename from test/test_io/test_supera/test_supera_model.py rename to test/test_io/test_pimm/test_pimm_model.py index 23aa083e..ce32f713 100644 --- a/test/test_io/test_supera/test_supera_model.py +++ b/test/test_io/test_pimm/test_pimm_model.py @@ -1,6 +1,6 @@ """Opt-in real-file training smoke test using models available on main. -Run with SPINE_SUPERA_SAMPLE pointing at a pysupera 4.1 production file. +Run with SPINE_PIMM_ROOT pointing at paired hits/labl directories. """ import os @@ -10,19 +10,19 @@ from .conftest import round_trip_cache -SAMPLE = os.environ.get("SPINE_SUPERA_SAMPLE") +SAMPLE = os.environ.get("SPINE_PIMM_ROOT") pytestmark = pytest.mark.skipif( - not SAMPLE, reason="Set SPINE_SUPERA_SAMPLE for real-file training" + not SAMPLE, reason="Set SPINE_PIMM_ROOT for real-file training" ) -def test_real_supera_forward_backward(tmp_path): +def test_real_pimm_forward_backward(tmp_path): """Backpropagate every training objective through the complete real event.""" torch = pytest.importorskip("torch") pytest.importorskip("MinkowskiEngine") pytest.importorskip("torch_cluster") from spine.io.collate import CollateAll - from spine.io.dataset import SuperaDataset + from spine.io.dataset import PIMMDataset from spine.model.full_chain.providers.segmentation import SegmentationStage from spine.model.full_chain.state import ChainState from spine.model.graph_spice import GraphSPICE, GraphSPICELoss @@ -38,7 +38,13 @@ def test_real_supera_forward_backward(tmp_path): ("ppn_label", "particle_points"), ) } - ds = SuperaDataset(file_keys=SAMPLE, dtype="float32", schema=schema, n_entry=1) + ds = PIMMDataset( + data_root=SAMPLE, + dataset_name="sim_pixel", + dtype="float32", + schema=schema, + n_entry=1, + ) e = ds[0] np.testing.assert_array_equal( e["data"].coordinate_data, e["seg_label"].coordinate_data diff --git a/test/test_io/test_pimm/test_reader.py b/test/test_io/test_pimm/test_reader.py new file mode 100644 index 00000000..526d04f2 --- /dev/null +++ b/test/test_io/test_pimm/test_reader.py @@ -0,0 +1,159 @@ +"""Exercise the real pimm-data backend, including worker serialization.""" + +import json +import pickle + +import h5py +import numpy as np +import pytest + +from spine.io.dataset import PIMMDataset +from spine.io.read import PIMMReader + +from .conftest import FULL_SCHEMA + + +@pytest.fixture +def corpus(tmp_path, raw): + """Use upstream's synthetic writer instead of mocking its label joins.""" + pytest.importorskip("pimm_data") + from pimm_data.testing import make_jaxtpc_sample + + root = str(tmp_path / "corpus") + make_jaxtpc_sample(root, readout_type="pixel", n_events=2, n_files=2) + for path in (tmp_path / "corpus" / "labl").glob("*.h5"): + with h5py.File(path, "r+") as file: + meta = raw["pimm"]["metadata"] + file["config"].attrs["supera_pixel_geometry"] = json.dumps( + meta["pixel_geometry"] * 2 + ) + file["config"].attrs["supera_hit_x_from"] = "nominal" + file["config"].attrs["supera_hit_energy"] = "charge" + return root + + +def test_reader_selection_pickle_and_labels(corpus): + """Selection crosses files without changing event or particle identity.""" + reader = PIMMReader(data_root=corpus, dataset_name="sim", entry_list=[3, 0, 2]) + first = reader[0] + assert first["file_index"] == 1 + assert first["file_entry_index"] == 1 + assert first["source_file_entry_index"] == 1 + assert ( + first["pimm"]["identity"]["event_id"] + != reader[1]["pimm"]["identity"]["event_id"] + ) + restored = pickle.loads(pickle.dumps(reader)) + np.testing.assert_array_equal( + first["pimm"]["hits"]["energy"], restored[0]["pimm"]["hits"]["energy"] + ) + reader.close() + restored.close() + + +def test_dataset_factories_and_grid(corpus): + """The public pimm provider parses the combined event through common machinery.""" + from spine.io.factories import dataset_factory, reader_factory + + cfg = dict(provider="pimm", data_root=corpus, dataset_name="sim") + reader = reader_factory(cfg) + assert isinstance(reader, PIMMReader) + reader.close() + ds = dataset_factory(dict(cfg, schema=FULL_SCHEMA), dtype="float32") + event = ds[0] + assert len(event["data"].features) + np.testing.assert_array_equal(event["data"].coords, event["seg_label"].coords) + assert np.isnan(event["particles"][0].energy_deposit) + assert set(ds[0]) == set(ds.__getitems__([0])[0]) + ds.reader.close() + with pytest.raises(ValueError, match="same voxel"): + PIMMDataset( + data_root=corpus, + schema={ + "data": dict(provider="tensor", voxel_size_mm=1), + "seg_label": dict(provider="segmentation", voxel_size_mm=3), + }, + dtype="float32", + ) + + +def test_invalid_projection(): + """Reject legacy raw products rather than silently reading truth deposits.""" + with pytest.raises(ValueError, match="aligned"): + PIMMReader(keys=["points"]) + + +def test_missing_dependency(monkeypatch): + """Base SPINE imports stay lightweight and missing extras fail clearly.""" + from spine.io.read import pimm + + def missing(name): + raise ImportError(name) + + monkeypatch.setattr(pimm, "import_module", missing) + with pytest.raises(ImportError, match=r"spine\[pimm\]"): + PIMMReader() + + +def test_missing_optional_metadata(corpus, monkeypatch, tmp_path): + """Absent event identity or geometry is not replaced with fabricated values.""" + reader = PIMMReader(data_root=corpus, dataset_name="sim") + monkeypatch.setattr(reader.dataset.hits_reader, "event_id_of", lambda _: None) + assert set(reader[0]["pimm"]["identity"]) == {"entry"} + reader.close() + path = tmp_path / "metadata.h5" + with h5py.File(path, "w") as file: + file.create_group("config") + assert PIMMReader._read_metadata(str(path))["pixel_geometry"] is None + + +def test_plane_local_groups(corpus): + """The final-layout per-plane group namespace is resolved by pimm-data.""" + from pathlib import Path + + from pimm_data._shard_meta import clear_cache + + before = PIMMReader(data_root=corpus, dataset_name="sim") + expected = before[0]["pimm"]["hits"] + before.close() + hits_path = next((Path(corpus) / "hits").glob("*.h5")) + labl_path = Path(corpus) / "labl" / hits_path.name.replace("_hits_", "_labl_") + with h5py.File(hits_path, "r+") as hits, h5py.File(labl_path, "r+") as labels: + hits.attrs["group_id_space"] = "plane" + for key in (k for k in hits if k.startswith("entry_")): + for volume in (k for k in hits[key] if k.startswith("volume_")): + plane = hits[f"{key}/{volume}/Pixel"] + old = plane["group_ids"][:] + # Reverse local numbering to make an accidental direct join wrong. + plane["group_ids"][:] = np.arange(len(old))[::-1] + target = labels[f"{key}/{volume}"].require_group("plane_Pixel") + target["group"] = old[::-1] + clear_cache() + after = PIMMReader(data_root=corpus, dataset_name="sim") + actual = after[0]["pimm"]["hits"] + for name in ( + "coord", + "energy", + "instance_fragment", + "instance_particle", + "segment_motif", + ): + np.testing.assert_array_equal(actual[name], expected[name]) + after.close() + + +def test_runs_and_explicit_files(corpus, tmp_path): + """Task discovery and explicit modality paths select the same events.""" + from pathlib import Path + + root = Path(corpus) + files = {key: str(root / key / "*.h5") for key in ("hits", "labl")} + explicit = PIMMReader(files=files, dataset_name="sim", split="") + identity = explicit[0]["pimm"]["identity"] + explicit.close() + task = tmp_path / "runs" / "task" + task.parent.mkdir() + root.rename(task) + runs = PIMMReader(data_root=str(task.parent), runs=["task"], dataset_name="sim") + assert runs[0]["pimm"]["identity"] == identity + runs.close() diff --git a/test/test_io/test_pimm/truth_helpers.py b/test/test_io/test_pimm/truth_helpers.py new file mode 100644 index 00000000..692e25d0 --- /dev/null +++ b/test/test_io/test_pimm/truth_helpers.py @@ -0,0 +1,89 @@ +"""Physical truth round-trip assertions shared by integration tests.""" + +import copy + +import numpy as np + +from spine.construct import BuildManager +from spine.io.collate import CollateAll +from spine.io.read import HDF5Reader +from spine.io.unwrap import Unwrapper +from spine.io.write import HDF5Writer + + +def check_truth_round_trip(events, path): + """Build all truth levels in cm and retain their associations through HDF5. + + Parameters + ---------- + events : list[dict] + Parsed events with the complete native schema and source provenance. + path : pathlib.Path + Temporary output HDF5 file. + + Returns + ------- + dict + Unwrapped truth products used for physical validation by callers. + """ + batch = CollateAll(data_keys=events[0])(events) + truth = Unwrapper()(batch) + original = copy.deepcopy(truth) + BuildManager(True, True, True, mode="truth", units="cm")(truth) + keys = [ + "meta", + "event_info", + "points_label", + "depositions_label", + "truth_fragments", + "truth_particles", + "truth_interactions", + ] + writer = HDF5Writer(file_name=str(path), keys=keys, format_version=2) + writer(truth, {}) + writer.finalize() + writer.close() + reader = HDF5Reader(file_keys=str(path)) + for index, event in enumerate(events): + restored = reader[index] + BuildManager(True, True, True, mode="truth", units="cm")(restored) + # BuildManager converts parsed truth to cm; voxel centers use +0.5. + expected = event["meta"].to_cm( + original["clust_label"][index].coords, center=True + ) + np.testing.assert_allclose(restored["points_label"], expected) + assert np.isclose( + restored["depositions_label"].sum(), event["data"].features.sum() + ) + for key in ("truth_fragments", "truth_particles", "truth_interactions"): + assert len(restored[key]) == len(truth[key][index]) + for before, after in zip(truth[key][index], restored[key]): + assert after.units == "cm" + assert before.id == after.id + np.testing.assert_allclose(before.points, after.points) + for field in ( + "parent_id", + "children_id", + "orig_id", + "pid", + "is_primary", + "vertex", + "energy_deposit", + "fragment_ids", + "particle_ids", + ): + if hasattr(before, field): + np.testing.assert_allclose( + getattr(before, field), getattr(after, field) + ) + for particle in restored["truth_particles"]: + source = event["particles"][particle.orig_id] + assert particle.pid == source.pid + assert particle.is_primary == (source.interaction_primary > 0) + for particle in truth["truth_particles"][index]: + for fragment in particle.fragments: + assert fragment.particle_id == particle.id + for interaction in truth["truth_interactions"][index]: + for particle in interaction.particles: + assert particle.interaction_id == interaction.id + return truth diff --git a/test/test_io/test_supera/conftest.py b/test/test_io/test_supera/conftest.py deleted file mode 100644 index 2c8ad8d3..00000000 --- a/test/test_io/test_supera/conftest.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Shared native Supera fixtures and complete schema.""" - -import h5py -import numpy as np -import pytest - -FULL_SCHEMA = { - key: {"provider": provider} - for key, provider in ( - ("data", "tensor"), - ("seg_label", "segmentation"), - ("clust_label", "cluster_label"), - ("ppn_label", "particle_points"), - ("particles", "particle"), - ("coord_label", "particle_coords"), - ("meta", "meta"), - ("graph_label", "particle_graph"), - ("source_ids", "source_ids"), - ("event_info", "event_info"), - ) -} - - -@pytest.fixture -def sample(tmp_path): - """Two events with sparse IDs, overlap, LE in a track and empty truth.""" - path = tmp_path / "supera.h5" - ids = np.array([10, 80, 99, 120, 150], dtype=np.int32) - # Track, shower, LE fragment of track, delta, Michel. - table = dict( - id=ids, - parent_id=ids, - ancestor_id=ids, - frag_id=ids, - inst_id=np.array([10, 80, -1, 120, 150]), - frag_inst_id=np.array([10, 80, 10, 120, 150]), - sem_type=np.array([2, 1, 5, 3, 4]), - frag_sem_type=np.array([2, 1, 5, 3, 4]), - inst_sem_type=np.array([2, 1, -1, 3, 4]), - pc_start=np.array([0, 3, 2, 4, 5]), - pc_end=np.array([2, 4, 3, 5, 6]), - frag_pc_start=np.array([0, 3, 2, 4, 5]), - frag_pc_end=np.array([2, 4, 3, 5, 6]), - inst_pc_start=np.array([0, 3, -1, 4, 5]), - inst_pc_le_start=np.array([2, 4, -1, 5, 6]), - inst_pc_end=np.array([3, 4, -1, 5, 6]), - pdg=np.array([13, 11, 11, 11, 11]), - parent_pdg=np.array([13] * 5), - geant4_trackid=ids, - interaction_id=np.zeros(5, dtype=int), - mass=np.ones(5), - ) - for key in ( - "start_pos", - "end_pos", - "first_pos", - "last_pos", - "start_mom", - "end_mom", - ): - table[key] = np.tile([1.5, 1.5, 1.5, 10.0], (5, 1)) - table["last_pos"][0, :3] = [4.5, 1.5, 1.5] - table["first_pos"][3, :3] = [10.5, 1.5, 1.5] - table["first_pos"][4, :3] = [13.5, 1.5, 1.5] - table["last_pos"][1] = np.nan - points = np.array( - [ - [1.5, 1.5, 1.5, 5, 2, 0], - [4.5, 1.5, 1.5, 2, 3, 0], - [7.5, 1.5, 1.5, 1, 4, 0], - [1.5, 1.5, 1.5, 0, 9, 0], - [10.5, 1.5, 1.5, 1, 5, 0], - [13.5, 1.5, 1.5, 1, 6, 0], - ], - dtype=np.float32, - ) - with h5py.File(path, "w") as file: - file["format_version"] = "4.1.0" - file["n_events"] = 3 - file.attrs["voxel_size"] = "3.0" - file.attrs["hit_energy"] = "charge" - file["events/offsets"] = [0, 5, 10, 10] - file["events/event_id"] = [11, 12, 13] - file["points/offsets"] = [0, 6, 12, 12] - file["points/flat"] = np.concatenate([points, points]) - file["points/columns"] = np.array( - ["x", "y", "z", "t", "dE", "dX"], dtype=h5py.string_dtype() - ) - for key, val in table.items(): - second = np.where(val >= 0, val + 6, val) if "pc_" in key else val - file["particles/" + key] = np.concatenate([val, second]) - file["inter/offsets"] = [0, 1, 2, 2] - file["inter/interaction_id"] = [0, 0] - file["inter/part_start"] = [0, 5] - file["inter/part_end"] = [5, 10] - file["inter/pc_start"] = [0, 6] - file["inter/pc_end"] = [6, 12] - file["groups/offsets"] = [0, 9, 18, 18] - file["groups/fragment_id"] = [10] * 18 - file["groups/is_le"] = [False] * 18 - file["groups/volume_offsets_index"] = [0, 2, 4, 6] - file["groups/volume_offsets"] = [0, 4, 0, 4, 0, 0] - # A huge virtual dataset: a projected read must never access it. - hit_plane = file.create_group("hit_labels/volume0/plane0") - hit_plane["offsets"] = [0, 2, 4, 4] - hit_plane.create_dataset( - "fragment_id", shape=(10**9,), dtype="i4", fillvalue=10 - ) - return str(path) - - -def round_trip_cache(batch, products, path, stage): - """Publish native products and reload them through cache collation. - - Parameters - ---------- - batch : dict - Collated input batch containing physical-source provenance. - products : dict - Batched products to cache, without source metadata. - path : pathlib.Path - Temporary cache repository. - stage : str - Cache stage name. - - Returns - ------- - dict - Re-collated products from the published cache stage. - """ - from spine.io.collate import CollateAll - from spine.io.read import CacheReader - from spine.io.unwrap import Unwrapper - from spine.io.write import CacheWriter - - metadata = { - key: value - for key, value in batch.items() - if key == "index" or key.startswith("source_file_") - } - payload = { - key: value.to_numpy() if hasattr(value, "to_numpy") else value - for key, value in products.items() - } - writer = CacheWriter(path=str(path), stage=stage, keys=list(products)) - writer(Unwrapper()({**metadata, **payload}), {}) - writer.finalize() - writer.close() - reader = CacheReader(path=str(path), stage=stage) - events = reader.get_many(list(range(len(batch["index"])))) - reader.close() - return CollateAll(data_keys=products)(events) diff --git a/test/test_io/test_supera/coverage.ini b/test/test_io/test_supera/coverage.ini deleted file mode 100644 index eb186c9a..00000000 --- a/test/test_io/test_supera/coverage.ini +++ /dev/null @@ -1,13 +0,0 @@ -# Strict coverage for native pysupera input support, including error paths. -[run] -branch = True -source = src/spine - -[report] -include = - src/spine/io/read/supera.py - src/spine/io/dataset/supera.py - src/spine/io/parse/supera/* -exclude_lines = -fail_under = 100 -show_missing = True diff --git a/test/test_io/test_supera/test_supera.py b/test/test_io/test_supera/test_supera.py deleted file mode 100644 index c59eaf7e..00000000 --- a/test/test_io/test_supera/test_supera.py +++ /dev/null @@ -1,319 +0,0 @@ -"""Native pysupera reader and aligned supervision contracts.""" - -import copy -import pickle - -import h5py -import numpy as np -import pytest - -from spine.io.collate import CollateAll -from spine.io.dataset import SuperaDataset -from spine.io.parse.supera import SuperaTensorParser -from spine.io.read import SuperaReader - -SCHEMA = { - key: {"provider": provider} - for key, provider in ( - ("data", "tensor"), - ("seg_label", "segmentation"), - ("clust_label", "cluster_label"), - ("ppn_label", "particle_points"), - ("particles", "particle"), - ) -} - - -def dataset(path, **kwargs): - """Build the complete parser schema for one synthetic input file.""" - return SuperaDataset(file_keys=path, dtype="float32", schema=SCHEMA, **kwargs) - - -def test_reader_projection_rebasing_multifile_and_pickle(sample, monkeypatch): - """Preserve projection, event selection and range bases across worker copies.""" - original = h5py.Dataset.__getitem__ - - def guarded(data_source, key): - """Fail if default training reads touch group or hit-label payloads.""" - assert not data_source.name.startswith("/hit_labels/") - assert not data_source.name.startswith("/groups/") - return original(data_source, key) - - monkeypatch.setattr(h5py.Dataset, "__getitem__", guarded) - reader = SuperaReader(file_keys=[sample, sample], entry_list=[4, 0, 2]) - first, second, empty = pickle.loads(pickle.dumps(reader)).get_many([0, 1, 2]) - assert first["file_index"] == 1 and first["file_entry_index"] == 1 - np.testing.assert_array_equal(first["points"], second["points"]) - np.testing.assert_array_equal( - first["particles"]["inst_pc_start"], second["particles"]["inst_pc_start"] - ) - assert first["inter"]["part_start"][0] == 0 - assert empty["points"].shape == (0, 6) - assert "groups" not in first and "hit_labels" not in first - - -def test_explicit_group_and_hit_slices(sample): - """Read only the selected event from independent group and hit domains.""" - reader = SuperaReader( - file_keys=sample, keys=["groups", "hit_labels/volume0/plane0"] - ) - event = reader[1] - assert len(event["groups"]["fragment_id"]) == 9 - assert len(event["hit_labels/volume0/plane0"]["fragment_id"]) == 2 - np.testing.assert_array_equal(event["groups"]["volume_offsets"], [0, 4]) - - -def test_alignment_energy_semantics_and_endpoints(sample): - """Keep energy, semantic, fragment and endpoint supervision aligned.""" - event = dataset(sample)[1] - np.testing.assert_array_equal(event["data"].features[:, 0], [11, 3, 4, 5, 6]) - np.testing.assert_array_equal(event["seg_label"].features[:, 0], [1, 1, 4, 3, 2]) - np.testing.assert_array_equal(event["clust_label"].cluster_ids, [0, 0, 2, 3, 4]) - np.testing.assert_array_equal(event["clust_label"].group_ids, [0, 0, 0, 3, 4]) - np.testing.assert_array_equal( - event["clust_label"].shapes, event["seg_label"].features[:, 0] - ) - np.testing.assert_array_equal( - event["data"].coordinate_data, event["seg_label"].coordinate_data - ) - np.testing.assert_array_equal( - event["data"].coordinate_data, event["clust_label"].coords - ) - np.testing.assert_allclose(event["data"].meta.size, 0.3) - ppn = event["ppn_label"] - assert len(ppn.features) == 4 # Track start/end, delta start, Michel start. - np.testing.assert_array_equal(ppn.features[:, 0], [1, 1, 3, 2]) - np.testing.assert_array_equal(ppn.features[:, 2], [0, 1, 0, 0]) - assert np.isfinite(ppn.coordinate_data).all() - - -def test_empty_event_and_collation(sample): - """Collate empty events without changing event-local particle references.""" - data_source = dataset(sample) - empty = data_source[2] - assert empty["data"].features.shape == (0, 1) - assert empty["ppn_label"].features.shape == (0, 3) - batch = CollateAll(data_keys=SCHEMA)([data_source[0], empty, data_source[1]]) - np.testing.assert_array_equal(batch["data"].counts, [5, 0, 5]) - np.testing.assert_array_equal( - batch["clust_label"].shapes.tensor, batch["seg_label"].features.tensor[:, 0] - ) - assert batch["clust_label"].particle_indexes.tensor[-1] == 4 - assert batch["ppn_label"].features.tensor[-1, 1] == 4 - - -def test_invalid_version_offsets_ranges_and_grid(sample): - """Reject incompatible files, corrupt ranges and inconsistent parser grids.""" - raw = SuperaReader(file_keys=sample)[0] - raw["particles"]["frag_pc_end"][0] = 4 - with pytest.raises(ValueError, match="overlapping"): - SuperaTensorParser("float32")(raw) - schema = copy.deepcopy(SCHEMA) - schema["data"]["voxel_size_mm"] = 6 - with pytest.raises(ValueError, match="same voxel"): - SuperaDataset(file_keys=sample, dtype="float32", schema=schema) - with h5py.File(sample, "r+") as file: - file["points/offsets"][1] = 100 - with pytest.raises(TypeError, match="offsets"): - SuperaReader(file_keys=sample) - with h5py.File(sample, "r+") as file: - file["format_version"][()] = "3.0.0" - with pytest.raises(TypeError, match="4.1.0"): - SuperaReader(file_keys=sample) - - -def test_winner_uses_time_then_source_order(sample): - """Resolve equal-class overlaps deterministically without losing energy.""" - raw = SuperaReader(file_keys=sample)[0] - # Make the shower a second track in the same cell. Earlier time wins. - raw["particles"]["inst_sem_type"][1] = 2 - from spine.io.parse.supera import SuperaClusterLabelParser - - parser = SuperaClusterLabelParser("float32") - event = parser(raw) - assert event.cluster_ids[0] == 1 - assert event.values[0] == 11 - raw.pop("_supera_prepared") - raw["points"][3, 3] = raw["points"][0, 3] - assert parser(raw).cluster_ids[0] == 0 - - -def test_unknown_and_dangling_relationships_rejected(sample): - """Reject unsupported semantics and particle links with no target row.""" - raw = SuperaReader(file_keys=sample)[0] - raw["particles"]["inst_sem_type"][0] = 6 - with pytest.raises(ValueError, match="Unknown Supera semantic"): - SuperaTensorParser("float32")(raw) - raw = SuperaReader(file_keys=sample)[0] - raw["particles"]["frag_inst_id"][0] = 999 - with pytest.raises(ValueError, match="Dangling"): - SuperaTensorParser("float32")(raw) - - -def test_grid_transform_and_particle_energy(sample): - """Preserve detector transforms and distinguish source from merged energy.""" - schema = copy.deepcopy(SCHEMA) - for parser in schema.values(): - parser["voxel_size_mm"] = [3, 3, 3] - parser["origin_mm"] = [9, -12, 3] - event = SuperaDataset(file_keys=sample, dtype="float32", schema=schema)[0] - np.testing.assert_allclose(event["ppn_label"].coordinate_data[0], [0.5, 0.5, 0.5]) - np.testing.assert_allclose(event["data"].meta.lower, [0, 0, 0]) - # The winning track sees 11 MeV in the overlapped input cell, but its - # source-fragment truth includes only its own 2 + 3 MeV contributions. - # The separate LE fragment contributes 4 MeV when building the truth group. - assert event["particles"][0].energy_deposit == 5 - - -def test_registered_factories(sample): - """Register native Supera readers and datasets under the same provider.""" - from spine.io.factories import dataset_factory, reader_factory - - assert isinstance( - reader_factory({"provider": "supera", "file_keys": sample}), SuperaReader - ) - data_source = dataset_factory( - {"provider": "supera", "file_keys": sample, "schema": SCHEMA}, dtype="float32" - ) - assert isinstance(data_source, SuperaDataset) - assert len(data_source) == 3 - - -@pytest.mark.parametrize("origin", [(0, 0), (0, np.nan, 0)]) -def test_reject_invalid_origin(origin): - """Reject malformed or nonfinite grid origins before reading events.""" - with pytest.raises(ValueError, match="origin_mm"): - SuperaTensorParser("float32", origin_mm=origin) - - -@pytest.mark.parametrize("size", [0, -1, np.inf]) -def test_reject_invalid_explicit_voxel_size(size): - """Reject invalid explicit grid dimensions at parser construction.""" - with pytest.raises(ValueError, match="voxel_size_mm"): - SuperaTensorParser("float32", voxel_size_mm=size) - - -@pytest.mark.parametrize("size", [None, "0", "[3, -1, 3]", np.nan]) -def test_reject_invalid_file_voxel_size(sample, size): - """Validate missing, encoded and numeric voxel-size attributes.""" - raw = SuperaReader(file_keys=sample)[0] - raw["metadata"]["voxel_size"] = size - with pytest.raises(ValueError, match="voxel_size|voxel size"): - SuperaTensorParser("float32")(raw) - - -@pytest.mark.parametrize( - ("column", "row", "value", "message"), - [ - ("id", 1, 10, "unique"), - ("id", 1, -1, "nonnegative"), - ("inst_pc_le_start", 0, 4, "LE range"), - ("frag_pc_end", 0, 1, "cover every point"), - ("frag_inst_id", 0, 80, "ownership"), - ("frag_sem_type", 2, 2, "semantics"), - ], -) -def test_reject_inconsistent_particle_table(sample, column, row, value, message): - """Reject corrupt identity, range and cross-level association fields.""" - raw = SuperaReader(file_keys=sample)[0] - raw["particles"][column][row] = value - with pytest.raises(ValueError, match=message): - SuperaTensorParser("float32")(raw) - - -@pytest.mark.parametrize("column", [0, 3, 4]) -def test_reject_nonfinite_point_values(sample, column): - """Reject nonfinite coordinates, times and deposited energy.""" - raw = SuperaReader(file_keys=sample)[0] - raw["points"][0, column] = np.nan - with pytest.raises(ValueError, match="Nonfinite"): - SuperaTensorParser("float32")(raw) - - -def test_empty_representative_retains_particle_row(sample): - """Keep truth rows for a representative whose point ranges are absent.""" - from spine.io.parse.supera import SuperaParticleParser - - raw = SuperaReader(file_keys=sample)[0] - raw["points"] = raw["points"][:5] - for key in ( - "pc_start", - "pc_end", - "frag_pc_start", - "frag_pc_end", - "inst_pc_start", - "inst_pc_end", - "inst_pc_le_start", - ): - raw["particles"][key][-1] = -1 - particles = SuperaParticleParser("float32")(raw) - assert len(particles) == 5 - assert particles[-1].num_voxels == 0 - assert particles[-1].energy_deposit == 0 - assert particles[-1].pdg_code == 11 - - -@pytest.mark.parametrize("kind", ["le", "nucleus", "nan", "below", "above"]) -def test_ppn_omits_ineligible_endpoints(sample, kind): - """Omit LE, nuclear, nonfinite and out-of-image instance targets.""" - from spine.io.parse.supera import SuperaParticlePointsParser - - raw = SuperaReader(file_keys=sample)[0] - table = raw["particles"] - if kind == "le": - table["inst_sem_type"][3] = 5 - table["frag_sem_type"][3] = 5 - table["inst_pc_le_start"][3] = table["inst_pc_start"][3] - elif kind == "nucleus": - table["pdg"][3] = 1000020040 - else: - table["first_pos"][3, 0] = {"nan": np.nan, "below": -1, "above": 100}[kind] - points = SuperaParticlePointsParser("float32")(raw) - assert len(points.features) == 3 - assert 3 not in points.features[:, 1] - - -def test_reader_without_compression_plugin(sample, monkeypatch): - """Uncompressed input remains usable without the optional filter package.""" - from spine.io.read import supera - - def missing_plugin(name): - """Simulate an absent optional dependency without changing other imports.""" - assert name == "hdf5plugin" - raise ImportError(name) - - monkeypatch.setattr(supera, "import_module", missing_plugin) - assert len(SuperaReader(file_keys=sample)[0]["points"]) == 6 - - -@pytest.mark.parametrize("key", ["unknown", "hit_labels", "hit_labels/volume0/plane9"]) -def test_reject_unknown_projection(sample, key): - """Require valid raw product names and explicit existing hit planes.""" - with pytest.raises(ValueError, match="Unsupported Supera product"): - SuperaReader(file_keys=sample, keys=[key]) - - -def test_reject_invalid_point_columns(sample): - """Validate declared point columns before any parser can misinterpret them.""" - with h5py.File(sample, "r+") as file: - file["points/columns"][0] = "wrong" - with pytest.raises(ValueError, match="unsupported point columns"): - SuperaReader(file_keys=sample) - - -def test_reject_group_in_place_of_dataset(sample): - """Reject malformed HDF5 structure with a clear dataset diagnostic.""" - with h5py.File(sample, "r+") as file: - del file["points/columns"] - file.create_group("points/columns") - with pytest.raises(TypeError, match="Expected a dataset"): - SuperaReader(file_keys=sample) - - -def test_raw_dataset_projection(sample): - """Expose raw selected products when no parser schema is supplied.""" - data = SuperaDataset(file_keys=sample, keys=["events"])[1] - assert data["events"]["event_id"] == 12 - assert "points" not in data - data = SuperaDataset(file_keys=sample)[0] - assert set(SuperaReader.default_keys).issubset(data) diff --git a/test/test_io/test_supera/test_supera_truth.py b/test/test_io/test_supera/test_supera_truth.py deleted file mode 100644 index c3f05b69..00000000 --- a/test/test_io/test_supera/test_supera_truth.py +++ /dev/null @@ -1,336 +0,0 @@ -"""Downstream truth contracts, provenance and physical-coordinate round trips.""" - -import copy - -import numpy as np - -from spine.construct import BuildManager -from spine.io.collate import CollateAll -from spine.io.dataset import SuperaDataset -from spine.io.parse.supera import ( - SuperaClusterLabelParser, - SuperaParticleCoordsParser, - SuperaParticleGraphParser, - SuperaParticleParser, -) -from spine.io.read import HDF5Reader, SuperaReader -from spine.io.unwrap import Unwrapper -from spine.io.write import HDF5Writer - -from .conftest import FULL_SCHEMA - - -def test_local_ids_coordinates_and_provenance(sample): - """Sparse source IDs cannot leak into row-indexed supervision, even in batches.""" - ds = SuperaDataset(file_keys=sample, schema=FULL_SCHEMA, dtype="float32") - events = [ds[i] for i in (0, 2, 1)] - batch = CollateAll(data_keys=ds.data_keys)(events) - unwrapped = Unwrapper()(batch) - for i, event in enumerate(events): - count = len(event["particles"]) - np.testing.assert_array_equal( - unwrapped["clust_label"][i].particle_field("particle"), np.arange(count) - ) - assert unwrapped["coord_label"][i].coordinate_data.shape == (count, 6) - np.testing.assert_allclose(unwrapped["meta"][i].size, 0.3) - np.testing.assert_array_equal( - unwrapped["source_ids"][i].features, event["source_ids"].features - ) - assert unwrapped["event_info"][i].features[0, 0] == [11, 13, 12][i] - if count: - coords = event["coord_label"].coordinate_data - np.testing.assert_allclose(coords[0], [0.5, 0.5, 0.5, 1.5, 0.5, 0.5]) - np.testing.assert_allclose(coords[1, :3], coords[1, 3:]) - assert event["source_ids"].features.max() > count - for row, particle in enumerate(event["particles"]): - labels = event["clust_label"].particles - assert particle.pid == labels["pid"][row] - assert particle.group_primary == labels["group_primary"][row] - assert ( - particle.interaction_primary == labels["interaction_primary"][row] - ) - - -def test_missing_endpoints_and_ancestry(sample): - """Missing links cannot wrap to the last row, and deposited endpoints are usable.""" - raw = SuperaReader(file_keys=sample)[0] - table = raw["particles"] - table["parent_id"][0] = -1 - table["ancestor_id"][0] = -1 - table["first_pos"][:2] = np.nan - table["last_pos"][0] = np.inf - table["first_pos"][2] = np.nan - table["last_pos"][2] = np.nan - # An unobserved extra row has neither endpoints nor a recoverable fragment. - for key, values in table.items(): - table[key] = np.concatenate((values, values[-1:])) - table["id"][-1] = 100000 - for key in ("parent_id", "ancestor_id", "frag_id", "inst_id", "frag_inst_id"): - table[key][-1] = -1 - for key in ("first_pos", "last_pos"): - table[key][-1] = np.nan - table["pc_start"][-1] = table["pc_end"][-1] = -1 - coords = SuperaParticleCoordsParser("float32")(raw) - assert np.isfinite(coords.coordinate_data[:5]).all() - assert np.isnan(coords.coordinate_data[-1]).all() - # Raw point times are microseconds; coordinate-label times are nanoseconds. - assert coords.features[0, 0] == 2000 - np.testing.assert_allclose(coords.coordinate_data[0, :3], [1.5, 0.5, 0.5]) - labels = SuperaClusterLabelParser("float32")(raw).particles - assert labels["interaction_primary"][0] == -1 - assert np.isnan(labels["vertex"][0]).all() - particles = SuperaParticleParser("float32")(raw) - assert particles[0].ancestor_track_id == -1 - assert np.isnan(particles[0].parent_position).all() - assert particles[-1].energy_deposit == 0 - assert SuperaParticleGraphParser("float32")(raw).index.shape == (2, 0) - - -def test_interaction_vertex_and_genealogy(sample): - """Stored interaction vertices override differing roots, with local graph links.""" - raw = SuperaReader(file_keys=sample)[0] - raw["inter"].update(x=np.array([30.0]), y=np.array([60.0]), z=np.array([90.0])) - raw["particles"]["parent_id"][3] = 10 - raw["particles"]["ancestor_id"][3] = 10 - raw["particles"]["geant4_parent_trackid"] = np.arange(5) + 200 - raw["particles"]["geant4_parent_pdg"] = np.full(5, 211) - labels = SuperaClusterLabelParser("float32")(raw).particles - np.testing.assert_allclose(labels["vertex"], np.tile([10, 20, 30], (5, 1))) - particles = SuperaParticleParser("float32")(raw) - assert particles[3].interaction_primary == 0 - assert particles[3].ancestor_track_id == 10 - assert particles[3].parent_track_id == 203 - assert particles[3].parent_pdg_code == 211 - np.testing.assert_array_equal(particles[0].children_id, [3]) - np.testing.assert_array_equal( - SuperaParticleGraphParser("float32")(raw).index, [[0], [3]] - ) - # A nonfinite stored vertex falls back to valid ancestry. - raw.pop("_supera_prepared") - raw["inter"]["x"][:] = np.nan - labels = SuperaClusterLabelParser("float32")(raw).particles - np.testing.assert_allclose(labels["vertex"][0], [0.5, 0.5, 0.5]) - - -def check_truth_round_trip(events, path): - """Build all truth levels in cm and retain their associations through HDF5. - - Parameters - ---------- - events : list[dict] - Parsed events with the complete native schema and source provenance. - path : pathlib.Path - Temporary output HDF5 file. - - Returns - ------- - dict - Unwrapped truth products used for physical validation by callers. - """ - batch = CollateAll(data_keys=events[0])(events) - truth = Unwrapper()(batch) - original = copy.deepcopy(truth) - BuildManager(True, True, True, mode="truth", units="cm")(truth) - keys = [ - "meta", - "event_info", - "source_ids", - "points_label", - "depositions_label", - "truth_fragments", - "truth_particles", - "truth_interactions", - ] - writer = HDF5Writer(file_name=str(path), keys=keys, format_version=2) - writer(truth, {}) - writer.finalize() - writer.close() - reader = HDF5Reader(file_keys=str(path)) - for index, event in enumerate(events): - restored = reader[index] - BuildManager(True, True, True, mode="truth", units="cm")(restored) - # BuildManager converts parsed truth to cm; voxel centers use +0.5. - expected = event["meta"].to_cm( - original["clust_label"][index].coords, center=True - ) - np.testing.assert_allclose(restored["points_label"], expected) - assert np.isclose( - restored["depositions_label"].sum(), event["data"].features.sum() - ) - for key in ("truth_fragments", "truth_particles", "truth_interactions"): - assert len(restored[key]) == len(truth[key][index]) - for before, after in zip(truth[key][index], restored[key]): - assert after.units == "cm" - assert before.id == after.id - np.testing.assert_allclose(before.points, after.points) - for field in ( - "parent_id", - "children_id", - "orig_id", - "pid", - "is_primary", - "vertex", - "energy_deposit", - "fragment_ids", - "particle_ids", - ): - if hasattr(before, field): - np.testing.assert_allclose( - getattr(before, field), getattr(after, field) - ) - for particle in restored["truth_particles"]: - source = event["particles"][particle.orig_id] - assert particle.pid == source.pid - assert particle.is_primary == (source.interaction_primary > 0) - for particle in truth["truth_particles"][index]: - for fragment in particle.fragments: - assert fragment.particle_id == particle.id - for interaction in truth["truth_interactions"][index]: - for particle in interaction.particles: - assert particle.interaction_id == interaction.id - return truth - - -def test_truth_round_trip(sample, tmp_path): - """Sparse IDs, multiple events and empty truth survive the full conversion path.""" - import h5py - - with h5py.File(sample, "r+") as file: - for row in (3, 8): - file["particles/parent_id"][row] = 10 - file["particles/ancestor_id"][row] = 10 - ds = SuperaDataset(file_keys=sample, schema=FULL_SCHEMA, dtype="float32") - truth = check_truth_round_trip([ds[i] for i in (0, 2, 1)], tmp_path / "truth.h5") - # Source energy is per stored particle; grouping must count each deposit once. - assert truth["truth_particles"][0][0].energy_deposit == 9 - np.testing.assert_array_equal(truth["truth_particles"][0][0].children_id, [1]) - assert truth["truth_particles"][0][1].parent_id == 0 - assert len(truth["truth_particles"][1]) == 0 - - -def downstream_batch(sample): - """Create two nodes per nonempty event, separated by an empty event. - - Parameters - ---------- - sample : str - Synthetic Supera file path. - - Returns - ------- - tuple[dict, IndexBatch] - Collated labels and fragment nodes with event-local particle references. - """ - from spine.data import IndexBatch - - ds = SuperaDataset(file_keys=sample, schema=FULL_SCHEMA, dtype="float32") - batch = CollateAll(data_keys=ds.data_keys)([ds[0], ds[2], ds[1]]) - clusts = IndexBatch( - [np.array([0, 1]), np.array([2]), np.array([5, 6]), np.array([7])], - counts=[2, 0, 2], - spans=[5, 0, 5], - single_counts=[2, 1, 2, 1], - ) - return batch, clusts - - -def test_closest_primary_batch(sample): - """Closest-primary selection must use local table rows, not sparse source IDs.""" - from spine.cluster.label import ( - get_cluster_closest_label_batch, - get_cluster_closest_primary_label_batch, - ) - from spine.data import TensorBatch - - batch, clusts = downstream_batch(sample) - primary = TensorBatch(np.ones(4, dtype=int), counts=[2, 0, 2]) - for select in ( - get_cluster_closest_label_batch, - get_cluster_closest_primary_label_batch, - ): - args = (batch["clust_label"], batch["coord_label"], clusts, primary) - selected = ( - select(*args, default=[0, 0]) - if select == get_cluster_closest_label_batch - else select(*args) - ) - np.testing.assert_array_equal(selected.numpy_tensor(), [1, 0, 1, 0]) - # A fragment's PID/interaction-primary describe its host, not its own electron. - assert ( - batch["clust_label"].particle_field("pid")[0][2] - == batch["clust_label"].particle_field("pid")[0][0] - ) - - -def test_orientation_batch(sample): - """Orientation targets follow particle-table offsets across empty batch entries.""" - import pytest - - torch = pytest.importorskip("torch") - from spine.data import TensorBatch - from spine.model.grappa.loss.node_orient import NodeOrientLoss - - batch, clusts = downstream_batch(sample) - starts = TensorBatch( - torch.tensor( - [[0.5, 0.5, 0.5], [2.5, 0.5, 0.5], [1.5, 0.5, 0.5], [2.5, 0.5, 0.5]] - ), - [2, 0, 2], - ) - ends = TensorBatch( - torch.tensor( - [[1.5, 0.5, 0.5], [2.5, 0.5, 0.5], [0.5, 0.5, 0.5], [2.5, 0.5, 0.5]] - ), - [2, 0, 2], - ) - target = NodeOrientLoss().materialize_target( - clust_label=batch["clust_label"].to_tensor(), - coord_label=batch["coord_label"].to_tensor(), - clusts=clusts, - start_points=starts, - end_points=ends, - ) - np.testing.assert_array_equal(target["target"].numpy_tensor(), [1, -1, 0, -1]) - np.testing.assert_array_equal( - target["valid"].numpy_tensor(), [True, False, True, False] - ) - - -def test_absorbed_fragment_uses_host_interaction(sample): - """A deposit absorbed from another interaction follows its host's training labels.""" - raw = SuperaReader(file_keys=sample)[0] - raw["particles"]["interaction_id"][2] = 1 - raw["inter"].update( - interaction_id=np.array([0, 1]), - x=np.array([30.0, 300.0]), - y=np.array([60.0, 600.0]), - z=np.array([90.0, 900.0]), - ) - labels = SuperaClusterLabelParser("float32")(raw).particles - assert labels["interaction"][2] == 0 - assert labels["pid"][2] == labels["pid"][0] - np.testing.assert_allclose(labels["vertex"][2], [10, 20, 30]) - particle = SuperaParticleParser("float32")(raw)[2] - assert particle.interaction_id == 0 - np.testing.assert_allclose(particle.ancestor_position, [10, 20, 30]) - - -def test_example_schema_is_complete(sample): - """The documented configuration provides all products without parser overrides.""" - from pathlib import Path - from textwrap import dedent - - import yaml - - # Exercise the published schema directly, without a standalone config file. - doc_path = Path(__file__).resolve().parents[3] / "docs/source/api/io.rst" - section = doc_path.read_text().split("Native pysupera input\n", 1)[1] - example = section.split(".. code-block:: yaml\n\n", 1)[1].split("\n\n", 1)[0] - config = yaml.safe_load(dedent(example)) - assert config["io"]["loader"]["minibatch_size"] == 1 - assert "batch_size" not in config["io"]["loader"] - schema = config["io"]["loader"]["dataset"]["schema"] - assert schema == FULL_SCHEMA - event = SuperaDataset(file_keys=sample, schema=schema, dtype="float32")[0] - BuildManager(True, True, True, mode="truth", units="cm")(event) - assert len(event["truth_particles"]) == 3 From af8e07a1f6d88eb5460e5cd996f301cd7b36154e Mon Sep 17 00:00:00 2001 From: Francois Drielsma Date: Mon, 9 Nov 2026 09:28:54 -0800 Subject: [PATCH 5/5] Remove obsolete input-format investigation and migration notes --- docs/source/api/io.rst | 24 +- docs/source/api/supera_format_findings.rst | 254 --------------------- src/spine/io/read/pimm.py | 2 +- 3 files changed, 8 insertions(+), 272 deletions(-) delete mode 100644 docs/source/api/supera_format_findings.rst diff --git a/docs/source/api/io.rst b/docs/source/api/io.rst index 2706d1f4..6400b83e 100644 --- a/docs/source/api/io.rst +++ b/docs/source/api/io.rst @@ -53,12 +53,11 @@ File Readers PIMM detector input ------------------- -Use ``provider: pimm`` for paired JAXTPC ``hits`` and pysupera-derived ``labl`` +Use ``provider: pimm`` for paired JAXTPC ``hits`` and ``labl`` files in the PIMM layout. ``PIMMReader`` delegates decoding, event alignment, per-hit overrides and particle-ID normalization to ``pimm_data.JAXTPCDataset``. SPINE consumes its untransformed NumPy output and retains its own parsers, -collation and augmentation. There is no direct pysupera HDF5 reader or -truth-deposit input fallback. +collation and augmentation. Install ``spine[pimm]``. The optional extra includes PyTorch, ``pimm-data`` and HDF5 compression filters; importing base SPINE does not import ``pimm-data``. @@ -94,10 +93,8 @@ For separately located shards, omit ``runs`` and provide ``files`` with ``hits`` and ``labl`` paths or glob patterns. The upstream loader requires matching task and event identities and rejects partial pairs. Standard SPINE entry selection (``n_entry``, ``n_skip``, ``entry_list``) is applied afterward. -Do not supply the old ``supera_pixel_*.h5`` as a ``labl`` file: prepare it with -upstream ``python -m pimm_data.supera_labl SUPERA HITS OUT`` first, or use the -published final-layout corpus. Conversion is an explicit preparation step, -never a hidden side effect of training. +Use the published final-layout corpus or prepare paired files using the +upstream PIMM data-preparation tools before starting training. Coordinates, charge and supervision ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -154,16 +151,9 @@ Truth construction uses the complete schema above with: Known limitations and validation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. toctree:: - :hidden: - - supera_format_findings - -This integration replaces the earlier truth-deposit prototype. Its old -training results do not establish detector-charge performance. Missing -fragment endpoints can reduce primary/orientation supervision; drift timing -can also displace nominal hit coordinates relative to true endpoints. -The reader must not silently repair either using truth deposits. +Missing fragment endpoints can reduce primary/orientation supervision. +Drift timing can also displace nominal hit coordinates relative to true +endpoints. Unavailable endpoints remain NaN. Synthetic tests cover alignment, charge conservation, ignored labels, empty clouds, missing endpoints, event selection, worker serialization, batching and diff --git a/docs/source/api/supera_format_findings.rst b/docs/source/api/supera_format_findings.rst deleted file mode 100644 index 744c99a5..00000000 --- a/docs/source/api/supera_format_findings.rst +++ /dev/null @@ -1,254 +0,0 @@ -pysupera 4.1 integration findings -================================= - -This report is prepared for the pysupera authors from the SPINE PR #191 -integration. It distinguishes an incorrect documented contract from intentional -loss of source detail and requests for a more explicit interchange schema. -The author responses below supersede the original open questions. - -.. note:: - - This is the historical investigation of the retired direct pysupera adapter. - The measurements remain useful, but descriptions of SPINE's deposit-based - energy attribution and endpoint fallbacks below do not describe the current - :ref:`PIMM integration `. - -Author clarification and integration direction ----------------------------------------------- - -The authors confirmed that the ``io_v3.py`` index docstring is outdated: -stored IDs are pre-filter in-memory particle indexes. They intend to change -these to stored row indexes. Own-particle point ranges describe deposits before -merging and are not expected to cover contributions from omitted particles. -Only LE contributions may merge across interaction boundaries. Self-parenting -is a supported primary criterion; ``geant4_parent_trackid == -1`` is the -preferred direct test. Fragment endpoint information is being added upstream. - -Crucially, detector training must read the separate JAXTPC hits, not -``points/flat``. The latter contains voxelized EDepSim energy in MeV; -``hit_energy=charge`` describes the hits and does not change that product. -Entry ``k`` of ``hit_labels/volumeV/plane0`` labels entry ``k`` of the matched -JAXTPC pixel hit array, with the source entry given by ``events/jaxtpc_entry``. - -SPINE is pivoting to a ``pimm`` provider using ``pimm-data`` for detector input -and pysupera for truth. The measurements below remain useful observations of -the original sample, but are not evidence of detector-charge training support. - -Reference production --------------------- - -* File: ``supera_pixel_j000000-000004.h5``, 1,000 events, format ``4.1.0``. -* Embedded software identity: ``drinkingkazu/pysupera`` commit - ``dab92d8fc850c09d0ca85cff16c6f36c28cf6d4b``, clean working tree. -* Producer: ``doraemon_prod.stages.supera``, EDepSim plus JAXTPC pixel input. -* Configuration: 3 mm stored voxels, ``hit_energy=charge``, - ``hit_x_from=nominal``, ``kinematics_steps=visible``. - -The source reference is `the pinned upstream tree -`_. -Measurements below use event 0, reading only that event's particle, interaction -and point slices. No hit-label payload was loaded. - -1. Sparse IDs contradict the documented row-index contract ----------------------------------------------------------- - -**Confirmed documentation/contract mismatch.** ``io_v3.py`` describes particle -IDs as event-local indexes into ``events/offsets[ev] + id``. The file instead -has 5,896 particle rows, with source IDs spanning 0 through 14,152. -``layout.py`` writes the retained original particle IDs and redirects parent -links in that same source-ID namespace. Sparse IDs are not themselves a format -error; interpreting them as rows is. - -SPINE originally normalized some relationships but retained source IDs in its -cluster ``particle`` label. Truth building then failed with -``Invalid particle ID found in fragment labels.`` That mixed-namespace adapter -bug is fixed by mapping all internal relationships to particle-table rows and -preserving source IDs in a separate provenance product. - -Requested upstream change: - -* Specify the namespace of every ID column, including ``frag_parent_id``, - ``inst_parent_id`` and hit/group labels. Publish the negative sentinel. -* Either write normalized row indexes plus a separate ``source_id``, or retain - sparse IDs and correct the reader documentation and examples to require a - source-ID-to-row map. Do not change the meaning silently within version 4.1. -* Validate uniqueness and nonnegative-ID referential integrity per event. - -2. Own-particle ranges do not partition the stored deposits ------------------------------------------------------------ - -**Observed loss of per-source attribution, consistent with selective row -retention in** ``layout.py``. The event has 28,789 stored point rows. The union -of the stored particles' ``pc_start:pc_end`` ranges covers only 15,334 of them, -without overlaps. The remaining 13,455 points contribute 4,686.0472 MeV of the -11,938.2904 MeV total. Summing only stored own-particle ranges gives -7,252.2432 MeV. - -Fragment and instance ranges do partition all point rows. ``layout.py`` retains -visible representatives and ancestry, not every contributing source particle. -The missing own-particle coverage therefore does **not** establish that the -file has lost deposit energy or that its fragment ranges are corrupt. It means -that complete per-source-particle energy cannot be recovered by summing those -stored own-particle ranges. - -SPINE accounts for all energy by integrating each fragment range exactly once -and assigning that contribution to its representative. Truth groups sum those -fragment contributions. This is fragment-attributed energy, not an assertion -that the representative's own Geant4 track deposited all of it. Overlap-winner -voxel sums and source-fragment energies are separate quantities. - -Requested upstream change: - -* Explicitly document that own-particle ranges may leave gaps, while fragment - and instance ranges are complete partitions. -* Store an explicit per-fragment/per-instance ``energy_deposit`` with units and - attribution semantics, or guarantee its derivation from the ranges. -* If complete source-particle truth is intended, retain the omitted contributing - rows or add a deposit-to-original-track association. SPINE cannot reconstruct - that information from fragment totals. - -3. Fragment endpoints are absent; deriving them is approximate --------------------------------------------------------------- - -**Missing information for fragment-level supervision, not a malformed array.** -There are 1,046 instance representatives, of which 815 have finite stored first -positions. No non-instance row has a finite ``first_pos``. After SPINE's common -grid and overlap selection, 270 non-LE fragments remain visible; 193 of these -lack stored first positions. - -``kinematics.py`` explicitly produces ``first_*``/``last_*`` only for instances. -They describe counted Geant4 step starts/ends, preferentially from particles -with non-LE membership. Showers intentionally have no last endpoint. -``start_*``/``end_*`` instead describe a track's creation/destruction and are -not interchangeable with the observed trajectory endpoints. - -SPINE needs one coordinate-label row per stored particle for primary matching -and orientation supervision. It retains stored collected-step endpoints when -available. Missing endpoints are derived from earliest/latest deposits in the -fragment, or the instance's non-LE range when applicable. These positions are -voxelized deposit locations: they cannot recover the exact original step -boundary or sub-voxel trajectory. Rows without usable deposits remain NaN; -non-track end labels repeat their start labels. - -Requested upstream change: - -* Provide collected-step endpoint coordinates/times for fragment representatives - as well as instances, with the same LE and visibility policy. -* Define validity independently for position, time and momentum. Record whether - a coordinate is a true step boundary, deposit midpoint or voxel center. -* State the intended policy for equal-time deposits and split tracks. An - explicitly versioned endpoint product would remove SPINE's approximation. - -4. Source interaction and host interaction can differ ------------------------------------------------------ - -**Two legitimate meanings currently share indirect representation.** -``layout.py`` explicitly permits particles' deposits to be absorbed into groups -from another interaction. Particle rows retain their own interaction; points -are ordered by their group's interaction. - -In event 0, 19 fragment rows have an ``interaction_id`` different from their -host instance. After overlap selection, 26 winning voxels have this mismatch. -Using the fragment row's own interaction for grouping supervision is therefore -incorrect for those voxels. SPINE uses the host instance's interaction for -cluster supervision and reconstructed truth-group membership. The stored -interaction vertex supplies a common vertex within that group. - -Requested upstream change: - -* Expose distinct ``source_interaction_id`` and ``instance_interaction_id`` - concepts, or document the required ``frag_inst_id`` join next to the field. -* State explicitly which interaction should supervise deposit/fragment/instance - grouping, and whether cross-interaction absorption is intended for training. -* Retain the original generator ``vertex_id`` and document its relation to the - event-local interaction table. - -5. Stored roots and true generator primaries need distinct definitions ----------------------------------------------------------------------- - -**Semantic ambiguity outside the retained genealogy.** ``parent_id`` points to -the nearest stored ancestor and becomes self when none is available; -``ancestor_id`` is the root of that redirected graph. Event 0 has 61 such roots -and 17 interactions. This graph is useful and walkable, but a stored root is -not, by definition alone, proof of generator-primary status. Real direct-parent -track/PDG columns are also available and should retain their separate meaning. - -SPINE preserves the stored genealogy, uses a host's self-parent status for -interaction-primary supervision, and propagates that label to its fragments. -Missing parents remain unknown. This convention should be reviewed by the -producer before treating it as an independent physics-primary definition. - -Requested upstream change: - -* Define and, preferably, store ``interaction_primary`` explicitly, including - the treatment of prompt decays and missing/unwritten ancestors. -* Distinguish redirected stored-parent links from original Geant4 parents. - Document self-root versus negative/missing sentinel behavior. -* Preserve full generator/neutrino truth if consumers are expected to recover - it. An interaction position alone cannot supply neutrino identity or missing - process information. - -6. Truth deposits, measured charge and association domains ----------------------------------------------------------- - -**Schema clarity request, not a discovered charge-conversion bug.** Despite -``hit_energy=charge``, the file's ``points/flat`` is truth-deposit dE in MeV, -not measured pixel charge. The measured hit payload remains in the separate -JAXTPC file. A model trained on this product is learning from truth-deposit -energy, not detector charge. Its learned weights are not thereby validated for -measured-charge inference. - -``groups`` has 54,863,490 rows over the production, versus 31,871,012 point -rows. The group domain is JAXTPC deposit groups concatenated over volumes, not -point rows. Hit labels occupy yet another, very large domain. Their offset -relationships must be explicit; equal-length or row-alignment assumptions are -unsafe. - -Requested upstream change: - -* Attach units, value kind (truth energy versus measured charge), coordinate - convention and row-domain metadata to each product, not only production - configuration attributes. -* If standalone detector-input training is intended, include the measured hit - payload or a stable source-file identity/entry/volume mapping, plus an - explicitly defined many-to-many hit-to-truth association. -* Publish a schema with per-product offsets, cardinality checks and examples - that read event slices rather than whole hit-label arrays. - -Minimal reproduction --------------------- - -This checks the ID/range findings without loading whole-file point or hit data: - -.. code-block:: python - - import h5py - import hdf5plugin # Register the file's compression filters. - import numpy as np - - with h5py.File("supera_pixel_j000000-000004.h5", "r") as f: - p0, p1 = f["events/offsets"][:2] - x0, x1 = f["points/offsets"][:2] - p = {key: value[p0:p1] for key, value in f["particles"].items()} - points = f["points/flat"][x0:x1] - columns = list(f["points/columns"].asstr()[:]) - cover = np.zeros(len(points), dtype=np.int32) - for start, stop in zip(p["pc_start"], p["pc_end"]): - if start >= 0: - cover[start - x0:stop - x0] += 1 - energy = points[:, columns.index("dE")] - print("particle rows, max ID:", len(p["id"]), p["id"].max()) - print("own-range coverage:", np.unique(cover, return_counts=True)) - print("total MeV:", energy.sum(dtype=np.float64)) - print("unattributed MeV:", energy[cover == 0].sum(dtype=np.float64)) - -Suggested upstream acceptance checks ------------------------------------- - -Before freezing the format, validate a fixture containing sparse source IDs, -absorbed/unwritten contributors, cross-interaction absorption, a split track, -a shower, missing endpoints and an empty event. Check ID namespaces, complete -fragment/instance point coverage, energy conservation, endpoint validity and -source-versus-host interaction semantics. Round-trip the fixture through the -upstream reader and a small independent consumer. Changing these contracts -should increment the format version. diff --git a/src/spine/io/read/pimm.py b/src/spine/io/read/pimm.py index aaa77836..653d1992 100644 --- a/src/spine/io/read/pimm.py +++ b/src/spine/io/read/pimm.py @@ -20,7 +20,7 @@ class PIMMReader(ReaderBase): The upstream dataset joins hits and ``labl`` by task and event identity. SPINE reads its untransformed NumPy products, preserving particle-table - row indexes until SPINE collation. No legacy pysupera file is read here. + row indexes until SPINE collation. """ name: ClassVar[str] = "pimm"