diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb0eea07..cd166d4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,7 +226,7 @@ jobs: -w /workspace \ "${COVERAGE_IMAGE}" \ bash -lc ' - python -m pip install --root-user-action=ignore -e . pytest pytest-cov coverage[toml] + python -m pip install --root-user-action=ignore -e ".[pimm]" pytest pytest-cov coverage[toml] mkdir -p /tmp/spine-sitecustomize printf "import h5py\n" > /tmp/spine-sitecustomize/sitecustomize.py NUMBA_DISABLE_JIT=1 PYTHONPATH="/tmp/spine-sitecustomize:${PYTHONPATH:-}" pytest test \ diff --git a/docker/spine/Dockerfile b/docker/spine/Dockerfile index 770428fb..c3b52dbe 100644 --- a/docker/spine/Dockerfile +++ b/docker/spine/Dockerfile @@ -184,9 +184,9 @@ RUN wget -O "${ICARUS_PHOTONLIB_PATH}" "${ICARUS_PHOTONLIB_URL}" && \ COPY pyproject.toml MANIFEST.in README.md ./ COPY src/ ./src/ -# Install SPINE with the optional visualization stack. The compiled model -# dependencies are installed explicitly above to keep their versions aligned. -RUN pip install --no-cache-dir --no-build-isolation .[viz] +# Install SPINE with PIMM readers and the optional visualization stack. +# Compiled model dependencies are installed above to keep versions aligned. +RUN pip install --no-cache-dir --no-build-isolation ".[viz,pimm]" # Verify that the default container environment and the explicit fallback setup # recipe both provide the compiled dependency imports SPINE needs at runtime. diff --git a/docs/source/api/io.rst b/docs/source/api/io.rst index 7deb26a8..6400b83e 100644 --- a/docs/source/api/io.rst +++ b/docs/source/api/io.rst @@ -44,9 +44,151 @@ File Readers :toctree: generated read.HDF5Reader + read.PIMMReader read.LArCVReader read.CacheReader +.. _pimm-input: + +PIMM detector input +------------------- + +Use ``provider: pimm`` for paired JAXTPC ``hits`` and ``labl`` +files in the PIMM layout. ``PIMMReader`` delegates decoding, event alignment, +per-hit overrides and particle-ID normalization to ``pimm_data.JAXTPCDataset``. +SPINE consumes its untransformed NumPy output and retains its own parsers, +collation and augmentation. + +Install ``spine[pimm]``. The optional extra includes PyTorch, ``pimm-data`` and +HDF5 compression filters; importing base SPINE does not import ``pimm-data``. +The Docker image installs this extra. The provisional dependency is pinned to +``doraemon-final-layout`` commit ``23dc94f3ac1134936891b281e4408a975d0097ed``; +an upstream release is required before this integration becomes official. +See the `upstream examples +`_. + +.. code-block:: yaml + + io: + loader: + minibatch_size: 1 + num_workers: 0 + dataset: + provider: pimm + data_root: /path/to/jaxtpc_pixel + dataset_name: sim_pixel + runs: "j*/*" + schema: + data: {provider: tensor} + seg_label: {provider: segmentation} + clust_label: {provider: cluster_label} + particles: {provider: particle} + coord_label: {provider: particle_coords} + ppn_label: {provider: particle_points} + meta: {provider: meta} + graph_label: {provider: particle_graph} + event_info: {provider: event_info} + +For separately located shards, omit ``runs`` and provide ``files`` with +``hits`` and ``labl`` paths or glob patterns. The upstream loader requires +matching task and event identities and rejects partial pairs. Standard SPINE +entry selection (``n_entry``, ``n_skip``, ``entry_list``) is applied afterward. +Use the published final-layout corpus or prepare paired files using the +upstream PIMM data-preparation tools before starting training. + +Coordinates, charge and supervision +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The 3D parsers currently require **pixel hits**. Upstream can read wire hits, +but a wire/tick pair does not define a unique 3D point; these parsers reject +wire input instead of interpreting its indexes as detector coordinates. + +Pixel indexes ``(py, pz, tick)`` are converted using ``supera_pixel_geometry`` +from the ``labl/config`` attributes. The transform uses pixel centers and +nominal drift coordinates, including the stored reference tick and drift +sign. ``supera_hit_x_from=nominal`` and ``supera_hit_energy=charge`` are required. +No interaction-time correction is inferred from truth. + +All spatial parsers share ``voxel_size_mm`` (default 3) and ``origin_mm`` +(default zero). Each event's integer grid is translated to nonnegative +coordinates; the shared ``Meta`` retains its absolute position in cm. +Conflicting parser grids are rejected. + +* The tensor contains **summed hit charge**, in the source hit units, not MeV. + Signed charge is retained. Coincident hits share one voxel. +* Semantic labels come directly from ``segment_motif``: shower, track, Michel, + delta, LE. Overlap winners use track, shower, delta, Michel, LE priority, + followed by source row order; every voxel product shares that selection. +* Voxels without any labelled contribution are excluded by default because + generic SPINE segmentation requires targets in 0--4. Charge from all hits + in retained voxels is preserved. Set ``include_unlabeled: true`` on every + spatial parser to retain target ``-1`` for inspection, not generic training. + LE remains class 4 with its upstream ownership references + (``le_is_stuff=False``); downstream objectives retain their normal selection. +* Fragment, instance, ancestor and graph references use event-local rows of + the PIMM particle table. SPINE collation applies the batch offsets. +* Coordinate labels contain one start/end pair per particle-table row. Stored + collected-step positions use the shared transform; missing endpoints remain + NaN. No deposit-derived or track-creation substitute is used. PPN includes + finite, in-image instance starts and track ends. +* Particle PID is mapped from PDG to SPINE's classes. Primary flags and + genealogy use the upstream table; interaction vertices use the host + instance's interaction. Missing ancestry is never indexed through ``-1``. +* Particle deposited energy is unavailable from these hit/label products and + remains NaN. Initial energy and momentum retain MeV and MeV/c. Generator + neutrino information is unavailable. The ``event_info`` product preserves + the upstream event ID when present and the source entry, without invented + run/subrun values. + +Truth construction uses the complete schema above with: + +.. code-block:: yaml + + build: + mode: truth + units: cm + +Known limitations and validation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Missing fragment endpoints can reduce primary/orientation supervision. +Drift timing can also displace nominal hit coordinates relative to true +endpoints. Unavailable endpoints remain NaN. + +Synthetic tests cover alignment, charge conservation, ignored labels, empty +clouds, missing endpoints, event selection, worker serialization, batching and +truth HDF5 round trips. Optional production checks use paired ``hits``/``labl`` +directories via ``SPINE_PIMM_ROOT`` and assert nonzero supervision and gradients +for segmentation, PPN, Graph-SPICE and shower/track/interaction GrapPA. Their +availability is not a claim that every objective has eligible targets in every +production. + +.. code-block:: bash + + PYTHONPATH=src pytest -q test/test_io/test_pimm + SPINE_PIMM_ROOT=/path/to/paired/data PYTHONPATH=src \ + pytest -q -s test/test_io/test_pimm + +.. autosummary:: + :toctree: generated + + parse.pimm.PIMMTensorParser + parse.pimm.PIMMSegmentationParser + parse.pimm.PIMMClusterLabelParser + parse.pimm.PIMMParticleParser + parse.pimm.PIMMParticlePointsParser + parse.pimm.PIMMParticleCoordsParser + parse.pimm.PIMMParticleGraphParser + parse.pimm.PIMMMetaParser + parse.pimm.PIMMEventInfoParser + dataset.pimm + read.pimm + parse.pimm.base + parse.pimm.cluster + parse.pimm.metadata + parse.pimm.particle + parse.pimm.tensor + Entry Filtering --------------- @@ -234,6 +376,7 @@ with raw input through the mixed dataset. dataset.LArCVDataset dataset.CacheDataset dataset.HDF5Dataset + dataset.PIMMDataset dataset.MixedDataset dataset.JointDataset diff --git a/pyproject.toml b/pyproject.toml index 75477265..07b0f8fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,13 @@ Documentation = "https://spine.readthedocs.io/" "Bug Tracker" = "https://github.com/DeepLearnPhysics/spine/issues" [project.optional-dependencies] +# Detector-hit readers and compression filters used by PIMM productions. +# Provisional doraemon-final-layout commit; require a release before adoption. +pimm = [ + "pimm-data @ git+https://github.com/OmarAlterkait/pimm-data.git@23dc94f3ac1134936891b281e4408a975d0097ed", + "hdf5plugin>=4", +] + # Visualization dependencies viz = [ "matplotlib>=3.3.0", diff --git a/src/spine/io/dataset/__init__.py b/src/spine/io/dataset/__init__.py index b0ba5b72..277c6270 100644 --- a/src/spine/io/dataset/__init__.py +++ b/src/spine/io/dataset/__init__.py @@ -15,6 +15,7 @@ from .joint import JointDataset from .larcv import LArCVDataset from .mixed import MixedDataset +from .pimm import PIMMDataset __all__ = [ "CacheDataset", @@ -22,4 +23,5 @@ "JointDataset", "LArCVDataset", "MixedDataset", + "PIMMDataset", ] diff --git a/src/spine/io/dataset/hdf5.py b/src/spine/io/dataset/hdf5.py index 4ef92280..12e8900b 100644 --- a/src/spine/io/dataset/hdf5.py +++ b/src/spine/io/dataset/hdf5.py @@ -30,6 +30,7 @@ class HDF5Dataset(BaseDataset): name: ClassVar[str] = "hdf5" supports_stages: ClassVar[bool] = False + parser_dict: ClassVar[dict[str, Any]] = PARSER_DICT parsers: dict[str, Any] reader: ReaderBase @@ -102,7 +103,7 @@ def __init__( parser_cfg = dict(parser_cfg) parser_stage = parser_cfg.pop("stage", stage) parser = instantiate( - PARSER_DICT, parser_cfg, alt_name="parser", dtype=dtype + self.parser_dict, parser_cfg, alt_name="parser", dtype=dtype ) self.parsers[data_product] = parser for key in parser.tree_keys: diff --git a/src/spine/io/dataset/pimm.py b/src/spine/io/dataset/pimm.py new file mode 100644 index 00000000..94bd1f85 --- /dev/null +++ b/src/spine/io/dataset/pimm.py @@ -0,0 +1,59 @@ +"""PIMM detector-hit dataset using the common HDF5 parsing machinery.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, ClassVar + +from spine.config.factory import module_dict + +from ..parse import pimm +from ..read.pimm import PIMMReader +from .hdf5 import HDF5Dataset + +__all__ = ["PIMMDataset"] + + +class PIMMDataset(HDF5Dataset): + """Parse aligned PIMM hits and truth into aligned SPINE training products.""" + + name: ClassVar[str] = "pimm" + parser_dict = module_dict(pimm) + + def _build_reader( + self, stage: str | None, stage_map: Mapping[str, str], kwargs: Mapping[str, Any] + ) -> PIMMReader: + """Select the native reader and enforce a common voxelization grid. + + Parameters + ---------- + stage : str, optional + Reserved cache-stage selection; always None for this dataset. + stage_map : mapping + Reserved cache product routing; empty for this dataset. + kwargs : mapping + Physical-reader and entry-selection options. + + Returns + ------- + PIMMReader + Native reader projected onto the products required by the schema. + + Raises + ------ + ValueError + If parsers request inconsistent grid settings. + """ + del stage, stage_map + + # Every supervision product must use the same voxel coordinates. + grids = {p.grid for p in self.parsers.values() if hasattr(p, "grid")} + if len(grids) > 1: + raise ValueError( + "PIMM parsers must use the same voxel size, origin and unlabelled policy." + ) + + # Reuse the parent dataset's schema-driven physical projection. + return PIMMReader( + keys=None if self.keys is None else tuple(self.keys), **kwargs + ) diff --git a/src/spine/io/parse/pimm/__init__.py b/src/spine/io/parse/pimm/__init__.py new file mode 100644 index 00000000..2072a6c9 --- /dev/null +++ b/src/spine/io/parse/pimm/__init__.py @@ -0,0 +1,6 @@ +"""Parsers for aligned PIMM detector hits and truth tables.""" + +from .cluster import * +from .metadata import * +from .particle import * +from .tensor import * diff --git a/src/spine/io/parse/pimm/base.py b/src/spine/io/parse/pimm/base.py new file mode 100644 index 00000000..72d58899 --- /dev/null +++ b/src/spine/io/parse/pimm/base.py @@ -0,0 +1,281 @@ +"""Shared PIMM hit voxelization and particle supervision.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.constants import PDG_TO_PID +from spine.data import Meta + +from ..base import ParserBase + + +def _shape(code: int | np.ndarray) -> np.ndarray: + """Map upstream one-based semantic codes to SPINE's five classes. + + Parameters + ---------- + code : int or np.ndarray + Upstream semantic codes; unsupported codes become -1. + + Returns + ------- + np.ndarray + SPINE class indexes with the same shape as the input. + """ + return np.array([-1, 0, 1, 3, 2, 4, -1])[np.clip(code, 0, 6)] + + +class PIMMParserBase(ParserBase): + """Share one hit grid and overlap selection across all training products. + + Pixel indexes are converted to nominal detector coordinates in mm, then + voxelized. Metadata uses cm. Charge is summed without energy conversion. + Label ties use track, shower, delta, Michel, LE, then source row order. + Voxels without labelled contributions are excluded by default. Charge + from every contribution is retained in a labelled voxel. Wire hits cannot + supply a unique 3D position. + """ + + def __init__( + self, + dtype: str, + voxel_size_mm: float | tuple[float, ...] = 3.0, + origin_mm: tuple[float, ...] = (0.0, 0.0, 0.0), + include_unlabeled: bool = False, + ) -> None: + """Configure the common detector grid. + + Parameters + ---------- + dtype : str + Output floating-point dtype. + voxel_size_mm : float or sequence[float], default 3 + Three physical voxel dimensions in mm. + origin_mm : sequence[float], default (0, 0, 0) + Detector grid origin in mm. + include_unlabeled : bool, default False + Retain voxels with target -1 for inspection. Generic SPINE + segmentation training requires these voxels to be excluded. + """ + super().__init__(dtype, pimm_event="pimm") + self.size = tuple(np.broadcast_to(np.asarray(voxel_size_mm, float), (3,))) + self.origin = tuple(np.asarray(origin_mm, float)) + if len(self.origin) != 3 or not np.isfinite(self.origin).all(): + raise ValueError("origin_mm must contain three finite values.") + if not np.isfinite(self.size).all() or min(self.size) <= 0: + raise ValueError("voxel_size_mm must be finite and positive.") + self.include_unlabeled = include_unlabeled + self.grid = (self.size, self.origin, include_unlabeled) + + def prepare(self, trees: dict[str, Any]) -> dict[str, Any]: + """Cache an aligned charge image, labels and particle-table references. + + Parameters + ---------- + trees : dict + Reader products containing the nested PIMM event. + + Returns + ------- + dict + Voxel coordinates, charge, truth references and shared metadata. + """ + cache = trees.setdefault("_pimm_prepared", {}) + if self.grid in cache: + return cache[self.grid] + raw = trees["pimm"] + hits, table = raw["hits"], raw["particle"] + points = self._coordinates(hits, raw["metadata"]) + values = np.asarray(hits["energy"]).reshape(-1) + semantic = np.asarray(hits["segment_motif"]).reshape(-1) + fragment = np.asarray(hits["instance_fragment"]).reshape(-1) + instance = np.asarray(hits["instance_particle"]).reshape(-1) + if any( + len(col) != len(points) for col in (values, semantic, fragment, instance) + ): + raise ValueError("PIMM hit and label row counts differ.") + if not np.isfinite(values).all() or not np.isfinite(points).all(): + raise ValueError("Nonfinite PIMM hit charge or coordinates.") + if np.any((semantic < -1) | (semantic > 4)): + raise ValueError("Invalid PIMM semantic labels.") + for refs in (fragment, instance): + if np.any((refs < -1) | (refs >= len(table["parent"]))): + raise ValueError("Invalid PIMM particle-table reference.") + + event = self._voxelize(points, values, semantic) + winner = event.pop("winner") + if not self.include_unlabeled: + keep = semantic[winner] >= 0 + winner = winner[keep] + event["coords"] = event["coords"][keep] + event["energy"] = event["energy"][keep] + event.update( + shape=semantic[winner], + fragment=fragment[winner], + instance=instance[winner], + columns=table, + ) + event["labels"] = self._particle_labels(event, raw["interaction"]) + cache[self.grid] = event + return event + + def _voxelize( + self, points: np.ndarray, values: np.ndarray, semantic: np.ndarray + ) -> dict[str, Any]: + """Sum coincident charge and select one shared label owner per voxel. + + Parameters + ---------- + points : np.ndarray + Physical hit coordinates in mm. + values : np.ndarray + Signed hit charge in the source units. + semantic : np.ndarray + Class labels used to resolve overlaps, including unknown (-1). + + Returns + ------- + dict + Sparse coordinates, summed charge, winning rows and cm metadata. + """ + size = np.asarray(self.size) + cells = np.floor((points - self.origin) / size).astype(np.int64) + lower_cell = cells.min(axis=0) if len(points) else np.zeros(3, dtype=np.int64) + coords, inverse = np.unique(cells - lower_cell, axis=0, return_inverse=True) + inverse = inverse.reshape(-1) + ranks = np.full(len(points), 5) + for rank, shape in enumerate((1, 0, 3, 2, 4)): + ranks[semantic == shape] = rank + order = np.lexsort((np.arange(len(points)), ranks, inverse)) + lower = (np.asarray(self.origin) + lower_cell * size) / 10.0 + count = coords.max(axis=0) + 1 if len(points) else np.ones(3, dtype=np.int64) + return { + "coords": coords, + "energy": np.bincount(inverse, weights=values), + "winner": ( + order[np.r_[True, np.diff(inverse[order]) != 0]] + if len(points) + else order + ), + "meta": Meta( + lower=lower, + upper=lower + count * size / 10.0, + size=size / 10.0, + count=count, + ), + } + + @staticmethod + def _coordinates(hits: dict, metadata: dict) -> np.ndarray: + """Convert pixel channel/tick indexes to nominal positions in mm. + + Parameters + ---------- + hits : dict + Upstream hit cloud in ``(py, pz, time)`` order. + metadata : dict + Per-volume pixel geometry copied by the PIMM label converter. + + Returns + ------- + np.ndarray + Detector ``(x, y, z)`` coordinates, one row per hit. + """ + if hits["readout_type"] != "pixel": + raise ValueError("PIMM 3D parsers require pixel hits; wire hits are 2D.") + geometry = metadata.get("pixel_geometry") + if geometry is None or metadata.get("hit_x_from") != "nominal": + raise ValueError( + "PIMM pixel input requires nominal pixel_geometry metadata." + ) + if metadata.get("hit_energy") != "charge": + raise ValueError("PIMM tensor input requires hit_energy=charge.") + coordinates = np.asarray(hits["coord"], dtype=float) + positions = np.empty_like(coordinates) + for plane_id, plane in enumerate(hits["planes"]): + volume = int(plane.split("_")[1]) + g = geometry[volume] + mask = np.asarray(hits["plane_id"]).reshape(-1) == plane_id + py, pz, ticks = coordinates[mask].T + positions[mask, 0] = ( + g["x_anode_mm"] + - g["drift_direction"] + * (ticks - g["reference_tick"]) + * g["mm_per_tick"] + ) + positions[mask, 1] = g["y_min_mm"] + (py + 0.5) * g["pitch_mm"] + positions[mask, 2] = g["z_min_mm"] + (pz + 0.5) * g["pitch_mm"] + return positions + + @staticmethod + def _particle_labels(event: dict, interactions: dict) -> dict[str, np.ndarray]: + """Build compact truth labels using upstream particle-table rows. + + Parameters + ---------- + event : dict + Prepared image and upstream particle table. + interactions : dict + Upstream interaction truth, including vertices in mm. + + Returns + ------- + dict + Particle sidecar used by fragment and graph supervision. + """ + table, meta = event["columns"], event["meta"] + rows = np.arange(len(table["parent"])) + host = table["fragment_instance"] + valid = (host >= 0) & (host < len(rows)) + interaction = np.full(len(rows), -1, dtype=np.int64) + pid = np.full(len(rows), -1, dtype=np.int64) + primary = np.full(len(rows), -1, dtype=np.int64) + interaction[valid] = table["interaction"][host[valid]] + pid[valid] = [ + PDG_TO_PID.get(abs(int(p)), -1) for p in table["pdg"][host[valid]] + ] + primary[valid] = table["is_primary"][host[valid]] + vertices = np.full((len(rows), 3), np.nan) + if "vertex" in interactions: + known = (interaction >= 0) & (interaction < len(interactions["vertex"])) + vertices[known] = interactions["vertex"][interaction[known]] + return { + "particle": rows, + "group": host.copy(), + "ancestor": table["ancestor"].copy(), + "interaction": interaction, + "nu": np.full(len(rows), -1), + "pid": pid, + "group_primary": np.where(valid, rows == host, -1), + "interaction_primary": primary, + "vertex": (vertices / 10.0 - meta.lower) / meta.size, + "momentum": np.linalg.norm(table["start_mom"][:, :3], axis=1), + "energy_init": table["start_mom"][:, 3].copy(), + "shape": table["motif"].copy(), + } + + def endpoints(self, trees: dict[str, Any]) -> tuple[np.ndarray, np.ndarray]: + """Return stored collected-step endpoints; missing values remain NaN. + + Parameters + ---------- + trees : dict + Nested PIMM event products. + + Returns + ------- + tuple[np.ndarray, np.ndarray] + Start/end positions in mm and times in ns. Non-track ends repeat + their starts. No truth-deposit or creation-point fallback is used. + """ + event = self.prepare(trees) + starts, ends = ( + event["columns"][key].copy() for key in ("first_pos", "last_pos") + ) + ends[event["labels"]["shape"] != 1] = starts[event["labels"]["shape"] != 1] + starts[~np.isfinite(starts)] = np.nan + ends[~np.isfinite(ends)] = np.nan + return starts, ends diff --git a/src/spine/io/parse/pimm/cluster.py b/src/spine/io/parse/pimm/cluster.py new file mode 100644 index 00000000..145f0c2a --- /dev/null +++ b/src/spine/io/parse/pimm/cluster.py @@ -0,0 +1,45 @@ +"""Compact fragmentation labels with event-local particle relationships.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.data import ClusterLabelData + +from .base import PIMMParserBase + +__all__ = ["PIMMClusterLabelParser"] + + +class PIMMClusterLabelParser(PIMMParserBase): + """Return fragment IDs and the winning fragment's particle table row.""" + + name = "cluster_label" + + def __call__(self, trees: dict[str, Any]) -> ClusterLabelData: + """Build compact fragment supervision with instance and ancestor links. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by PIMMReader. + + Returns + ------- + ClusterLabelData + Compact fragment labels and a named particle-truth sidecar. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + # The winning fragment row is also the compact particle association. + return ClusterLabelData( + coords=event["coords"].astype(self.itype), + features=np.column_stack( + (event["energy"], event["fragment"], event["fragment"]) + ).astype(self.ftype), + particles=event["labels"], + meta=event["meta"], + remove_duplicates=False, + ) diff --git a/src/spine/io/parse/pimm/metadata.py b/src/spine/io/parse/pimm/metadata.py new file mode 100644 index 00000000..55c75174 --- /dev/null +++ b/src/spine/io/parse/pimm/metadata.py @@ -0,0 +1,72 @@ +"""Shared image metadata and native event/particle provenance.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.data import Meta, TensorData + +from ..base import ParserBase +from .base import PIMMParserBase + +__all__ = ["PIMMMetaParser", "PIMMEventInfoParser"] + + +class PIMMMetaParser(PIMMParserBase): + """Expose the shared image transform for truth construction in cm.""" + + name = "meta" + + def __call__(self, trees: dict[str, Any]) -> Meta: + """Return the metadata shared by all native spatial products. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + Meta + Detector bounds and voxel dimensions in centimetres. + """ + return self.prepare(trees)["meta"] + + +class PIMMEventInfoParser(ParserBase): + """Preserve native event/job identities without fabricating run/subrun IDs.""" + + name = "event_info" + + def __init__(self, dtype: str) -> None: + """Request the small event-identity product. + + Parameters + ---------- + dtype : str + Dataset floating-point dtype; identities retain integer precision. + """ + super().__init__(dtype, pimm_event="pimm") + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return all stored identity fields as a named integer tensor. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + TensorData + One row containing the file's event, input and job identities. + """ + fields = sorted(trees["pimm"]["identity"]) + return TensorData( + features=np.array( + [[trees["pimm"]["identity"][key] for key in fields]], dtype=np.int64 + ), + feature_fields={key: (column,) for column, key in enumerate(fields)}, + ) diff --git a/src/spine/io/parse/pimm/particle.py b/src/spine/io/parse/pimm/particle.py new file mode 100644 index 00000000..1c1fff19 --- /dev/null +++ b/src/spine/io/parse/pimm/particle.py @@ -0,0 +1,268 @@ +"""Particle truth, coordinate labels, genealogy and PPN targets.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np + +from spine.data import EdgeIndexData, Meta, ObjectListData, Particle, TensorData + +from .base import PIMMParserBase, _shape + +__all__ = [ + "PIMMParticleParser", + "PIMMParticlePointsParser", + "PIMMParticleCoordsParser", + "PIMMParticleGraphParser", +] + + +class PIMMParticleParser(PIMMParserBase): + """Translate the full stored particle table, preserving event-local IDs.""" + + name = "particle" + + def __call__(self, trees: dict[str, Any]) -> ObjectListData: + """Return SPINE particle truth in voxel units with MeV kinematics. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by PIMMReader. + + Returns + ------- + ObjectListData + Complete stored particle table in voxel units with MeV kinematics. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table = event["columns"] + particles = [] + + # Hit charge is not a deposited-energy measurement. PIMM's label table + # does not provide per-fragment deposited energy; leave its NaN sentinel. + fragments = event["fragment"] + event_counts = np.bincount( + fragments[fragments >= 0], minlength=len(table["parent"]) + ) + + starts, ends = self.endpoints(trees) + + # Preserve the full table so PPN and cluster references remain valid. + for row in range(len(table["parent"])): + particle = Particle( + id=row, + parent_id=int(table["parent"][row]), + group_id=int(table["fragment_instance"][row]), + interaction_id=int(event["labels"]["interaction"][row]), + track_id=int(table["geant4_trackid"][row]), + pdg_code=int(table["pdg"][row]), + parent_pdg_code=int(table["parent_pdg"][row]), + shape=int(event["labels"]["shape"][row]), + energy_init=float(table["start_mom"][row, 3]), + momentum=table["start_mom"][row, :3].copy(), + pid=int(event["labels"]["pid"][row]), + group_primary=int(event["labels"]["group_primary"][row]), + interaction_primary=int(event["labels"]["interaction_primary"][row]), + ancestor_position=event["labels"]["vertex"][row].astype(self.ftype), + children_id=np.flatnonzero( + (table["parent"] == row) & (np.arange(len(table["parent"])) != row) + ).astype(np.int32), + end_momentum=table["end_mom"][row, :3].copy(), + units="px", + ) + + # Positions use voxel units; kinematic times remain in nanoseconds. + for positions, target in ( + (table["start_pos"], "position"), + (table["end_pos"], "end_position"), + (starts, "first_step"), + (ends, "last_step"), + ): + setattr( + particle, + target, + ( + (positions[row, :3] / 10.0 - event["meta"].lower) + / event["meta"].size + ).astype(self.ftype), + ) + particle.t, particle.end_t = float(table["start_pos"][row, 3]), float( + table["end_pos"][row, 3] + ) + particle.first_step_t, particle.last_step_t = float(starts[row, 3]), float( + ends[row, 3] + ) + self._fill_ancestry(particle, row, table, event["meta"]) + particle.num_voxels = int(event_counts[row]) + particles.append(particle) + return ObjectListData(particles, Particle()) + + def _fill_ancestry( + self, particle: Particle, row: int, table: dict[str, np.ndarray], meta: Meta + ) -> None: + """Populate available genealogy without dereferencing missing links. + + Parameters + ---------- + particle : Particle + Output truth particle, modified in place. + row : int + Event-local table row. + table : dict[str, np.ndarray] + Particle columns with normalized relationships. + meta : Meta + Shared detector transform in centimetres. + """ + # Missing links must never index the last particle through -1. + for relation in ("parent", "ancestor"): + relative = int(table[relation][row]) + if relative >= 0: + setattr( + particle, + relation + "_track_id", + int(table["geant4_trackid"][relative]), + ) + setattr(particle, relation + "_pdg_code", int(table["pdg"][relative])) + setattr( + particle, + relation + "_t", + float(table["start_pos"][relative, 3]), + ) + parent = int(table["parent"][row]) + if parent >= 0: + particle.parent_position = ( + (table["start_pos"][parent, :3] / 10.0 - meta.lower) / meta.size + ).astype(self.ftype) + if "geant4_parent_trackid" in table: + particle.parent_track_id = int(table["geant4_parent_trackid"][row]) + particle.parent_pdg_code = int(table["geant4_parent_pdg"][row]) + + +class PIMMParticlePointsParser(PIMMParserBase): + """PPN instance starts and track ends from collected-step kinematics.""" + + name = "particle_points" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Omit unknown, LE, nonfinite and out-of-image endpoints. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by PIMMReader. + + Returns + ------- + TensorData + Instance starts and track ends with shape, particle and endpoint labels. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + table, meta = event["columns"], event["meta"] + rows = [] + + # Only instances represented in the voxel image can provide targets. + for row in np.unique(event["instance"][event["instance"] >= 0]): + shape = int(_shape(table["instance_sem_type"][row])) + if shape not in (0, 1, 2, 3) or abs(table["pdg"][row]) > 1000000000: + continue + for endpoint, key in enumerate( + ("first_pos", "last_pos") if shape == 1 else ("first_pos",) + ): + point = (table[key][row, :3] / 10.0 - meta.lower) / meta.size + if ( + np.isfinite(point).all() + and np.all(point >= 0) + and np.all(point < meta.count) + ): + rows.append([*point, shape, row, endpoint]) + + # Preserve a stable six-column layout even when no endpoints survive. + data = np.asarray(rows, dtype=self.ftype).reshape(-1, 6) + return TensorData( + coords=data[:, :3], + features=data[:, 3:], + meta=meta, + coordinate_groups={"point": (0, 1, 2)}, + feature_fields={"shape": (0,), "particle": (1,), "endpoint": (2,)}, + index_cols=np.array([1]), + index_shifts=np.array([len(table["parent"])]), + ) + + +class PIMMParticleCoordsParser(PIMMParserBase): + """One pair of collected-step endpoints per stored particle-table row. + + Stored collected-step endpoints are used as provided by PIMM. Missing + coordinates and times remain NaN; no deposit-based approximation or + creation/destruction positions are substituted. Non-track ends repeat starts. + """ + + name = "particle_coords" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return aligned start/end coordinates and time/shape features. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + TensorData + Six coordinate columns and two features, with one row per particle. + Stored kinematic times remain in nanoseconds. + """ + event = self.prepare(trees) + meta = event["meta"] + starts, ends = self.endpoints(trees) + shapes = event["labels"]["shape"] + coords = np.column_stack( + ( + (starts[:, :3] / 10.0 - meta.lower) / meta.size, + (ends[:, :3] / 10.0 - meta.lower) / meta.size, + ) + ) + coords[~np.isfinite(coords)] = np.nan + return TensorData( + coords=coords.astype(self.ftype), + features=np.column_stack((starts[:, 3], shapes)).astype(self.ftype), + meta=meta, + coordinate_groups={"start": (0, 1, 2), "end": (3, 4, 5)}, + feature_fields={"time": (0,), "shape": (1,)}, + ) + + +class PIMMParticleGraphParser(PIMMParserBase): + """Directed stored-parent links in the event-local particle namespace. + + Truth construction contracts invisible rows and groups using this graph. + Retain every stored link so contraction can recover visible descendants. + """ + + name = "particle_graph" + + def __call__(self, trees: dict[str, Any]) -> EdgeIndexData: + """Return parent-to-child edges, excluding roots and missing parents. + + Parameters + ---------- + trees : dict[str, Any] + Projected native event products. + + Returns + ------- + EdgeIndexData + Directed edges with a span equal to the full particle-table length. + """ + table = self.prepare(trees)["columns"] + parents = table["parent"] + children = np.arange(len(parents)) + valid = (parents >= 0) & (parents != children) + return EdgeIndexData( + features=np.stack((parents[valid], children[valid])), span=len(children) + ) diff --git a/src/spine/io/parse/pimm/tensor.py b/src/spine/io/parse/pimm/tensor.py new file mode 100644 index 00000000..f925e828 --- /dev/null +++ b/src/spine/io/parse/pimm/tensor.py @@ -0,0 +1,65 @@ +"""Aligned input tensors and semantic segmentation labels.""" + +from __future__ import annotations + +from typing import Any + +from spine.data import TensorData + +from .base import PIMMParserBase + +__all__ = ["PIMMTensorParser", "PIMMSegmentationParser"] + + +class PIMMTensorParser(PIMMParserBase): + """Return voxelized detector hit charge.""" + + name = "tensor" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return summed detector charge on the shared sparse grid. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by PIMMReader. + + Returns + ------- + TensorData + Sparse coordinates and summed charge in the source hit units. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + return TensorData( + coords=event["coords"].astype(self.itype), + features=event["energy"].astype(self.ftype)[:, None], + meta=event["meta"], + ) + + +class PIMMSegmentationParser(PIMMParserBase): + """Return semantic labels in exactly the input voxel order.""" + + name = "segmentation" + + def __call__(self, trees: dict[str, Any]) -> TensorData: + """Return five-class labels; unlabelled hits use -1. + + Parameters + ---------- + trees : dict[str, Any] + Raw event products returned by PIMMReader. + + Returns + ------- + TensorData + Semantic labels sharing the input tensor coordinates and row order. + """ + # Reuse the same grid and overlap winners as the input tensor. + event = self.prepare(trees) + return TensorData( + coords=event["coords"].astype(self.itype), + features=event["shape"].astype(self.ftype)[:, None], + meta=event["meta"], + ) diff --git a/src/spine/io/read/__init__.py b/src/spine/io/read/__init__.py index 7abc6216..c640f3b0 100644 --- a/src/spine/io/read/__init__.py +++ b/src/spine/io/read/__init__.py @@ -3,5 +3,6 @@ from .cache import CacheReader from .hdf5 import HDF5Reader from .larcv import LArCVReader +from .pimm import PIMMReader -__all__ = ["CacheReader", "HDF5Reader", "LArCVReader"] +__all__ = ["CacheReader", "HDF5Reader", "LArCVReader", "PIMMReader"] diff --git a/src/spine/io/read/base.py b/src/spine/io/read/base.py index 3e4c618c..cbfe8a98 100644 --- a/src/spine/io/read/base.py +++ b/src/spine/io/read/base.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from collections.abc import Sequence from numbers import Real -from typing import Any +from typing import Any, ClassVar import numpy as np @@ -45,11 +45,12 @@ class ReaderBase(ABC): Maps each available (run, subrun, event) triplet onto an entry_index index """ - name: str + name: ClassVar[str] file_paths: list[str] file_index: np.ndarray file_offsets: np.ndarray entry_index: np.ndarray + eligible_entry_index: np.ndarray num_entries: int run_info: list[tuple[int, int, int]] | None = None run_map: dict[tuple[int, int, int], int] | None = None @@ -199,9 +200,7 @@ def is_remote_path(path: str) -> bool: bool `True` if the path is a supported remote URI """ - return isinstance(path, str) and ( - path.startswith("root://") or path.startswith("xroot://") - ) + return isinstance(path, str) and (path.startswith(("root://", "xroot://"))) def process_file_paths( self, diff --git a/src/spine/io/read/cache.py b/src/spine/io/read/cache.py index 964bc4ac..4be84686 100644 --- a/src/spine/io/read/cache.py +++ b/src/spine/io/read/cache.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import Any +from typing import Any, ClassVar import numpy as np @@ -24,7 +24,7 @@ class CacheReader(ReaderBase): cannot change the files observed by an active reader. """ - name = "cache" + name: ClassVar[str] = "cache" def __init__( self, diff --git a/src/spine/io/read/larcv.py b/src/spine/io/read/larcv.py index 995af09b..ec642b02 100644 --- a/src/spine/io/read/larcv.py +++ b/src/spine/io/read/larcv.py @@ -2,7 +2,7 @@ import os from collections.abc import Sequence -from typing import Any +from typing import Any, ClassVar import numpy as np @@ -33,7 +33,7 @@ class LArCVReader(ReaderBase): trees corresponding to each of the requested data products. """ - name: str = "larcv" + name: ClassVar[str] = "larcv" def __init__( self, diff --git a/src/spine/io/read/pimm.py b/src/spine/io/read/pimm.py new file mode 100644 index 00000000..653d1992 --- /dev/null +++ b/src/spine/io/read/pimm.py @@ -0,0 +1,159 @@ +"""Event access to aligned PIMM detector hits and label tables.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from importlib import import_module +from typing import Any, ClassVar + +import h5py +import numpy as np + +from .base import ReaderBase + +__all__ = ["PIMMReader"] + + +class PIMMReader(ReaderBase): + """Delegate detector decoding and truth associations to ``pimm-data``. + + The upstream dataset joins hits and ``labl`` by task and event identity. + SPINE reads its untransformed NumPy products, preserving particle-table + row indexes until SPINE collation. + """ + + name: ClassVar[str] = "pimm" + + # Public discovery options mirror the upstream dataset API. + # pylint: disable=too-many-arguments,too-many-positional-arguments + def __init__( + self, + data_root: str = ".", + files: Mapping[str, str | list[str]] | None = None, + runs: str | list[str] | None = None, + split: str | None = None, + dataset_name: str = "sim_pixel", + volume: int | None = None, + keys: Sequence[str] | None = None, + **selection: Any, + ) -> None: + """Index paired detector-hit and label shards. + + Parameters + ---------- + data_root : str, default "." + PIMM corpus root, containing modality or task directories. + files : mapping, optional + Explicit ``hits`` and ``labl`` paths or glob patterns. + runs : str or list[str], optional + Task directories or glob relative to the corpus root. + split : str, optional + Upstream split directory; defaults to flat discovery without runs. + dataset_name : str, default "sim_pixel" + Upstream file prefix. + volume : int, optional + Restrict input to one detector volume. + keys : sequence[str], optional + Only ``pimm`` is supported: it is one aligned upstream event. + **selection : dict + Standard ReaderBase entry selection. No run/subrun is invented. + """ + if keys is not None and set(keys) != {"pimm"}: + raise ValueError("PIMMReader exposes the aligned 'pimm' event product.") + try: + backend = import_module("pimm_data") + except ImportError as err: + raise ImportError( + "PIMM input requires the optional spine[pimm] extra." + ) from err + + # Keep all ownership references, including LE; semantic losses decide + # which classes to supervise. Never apply upstream transforms twice. + self.dataset = backend.JAXTPCDataset( + data_root=data_root, + files=files, + runs=runs, + split=("train" if runs is not None else "") if split is None else split, + dataset_name=dataset_name, + modalities=("hits",), + labels="all", + le_is_stuff=False, + volume=volume, + allow_partial=False, + ) + reader = self.dataset.hits_reader + self.file_paths = list(reader.h5_files) + counts = np.array([len(index) for index in reader.indices]) + self.num_entries = int(counts.sum()) + self.file_offsets = np.r_[0, np.cumsum(counts)[:-1]].astype(np.int64) + self.file_index = np.repeat(np.arange(len(counts)), counts) + self.process_entry_list(**selection) + + self.metadata = [ + self._read_metadata(path) for path in self.dataset.labl_reader.h5_files + ] + + @staticmethod + def _read_metadata(path: str) -> dict[str, Any]: + """Read the small detector-geometry attributes from a label shard. + + Parameters + ---------- + path : str + PIMM labl shard path. + + Returns + ------- + dict + Pixel geometry and coordinate/value conventions. + """ + with h5py.File(path, "r") as file: + attrs: dict[str, Any] = dict(file["config"].attrs) + geometry = attrs.get("supera_pixel_geometry") + return { + "pixel_geometry": json.loads(geometry) if geometry is not None else None, + "hit_x_from": attrs.get("supera_hit_x_from"), + "hit_energy": attrs.get("supera_hit_energy"), + } + + def get(self, idx: int) -> dict[str, Any]: + """Read one aligned event and its physical source identity. + + Parameters + ---------- + idx : int + Index in SPINE's selected entry list. + + Returns + ------- + dict + Nested ``pimm`` event and standard source provenance. + """ + entry = int(self.entry_index[idx]) + file_index, event_num = self.dataset.hits_reader.locate(entry) + labl_index, _ = self.dataset.labl_reader.locate(entry) + event = self.dataset.get_data(entry) + event["metadata"] = self.metadata[labl_index] + identity = self.dataset.hits_reader.event_id_of(entry) + event["identity"] = {"entry": event_num} + if identity is not None: + event["identity"]["event_id"] = identity + result = { + "index": idx, + "file_index": file_index, + "file_entry_index": event_num, + "pimm": event, + } + result.update(self.get_source_provenance(file_index, event_num)) + return result + + def close(self) -> None: + """Release upstream HDF5 handles before leaving a worker.""" + self.dataset.hits_reader.close() + self.dataset.labl_reader.close() + + def __getstate__(self) -> dict[str, Any]: + """Drop live handles before pickling for spawned loader workers.""" + self.close() + return self.__dict__.copy() diff --git a/test/test_io/test_pimm/__init__.py b/test/test_io/test_pimm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_io/test_pimm/conftest.py b/test/test_io/test_pimm/conftest.py new file mode 100644 index 00000000..756ab39a --- /dev/null +++ b/test/test_io/test_pimm/conftest.py @@ -0,0 +1,127 @@ +"""Small native PIMM events used to check SPINE adaptation.""" + +import numpy as np +import pytest + +FULL_SCHEMA = { + key: {"provider": provider} + for key, provider in ( + ("data", "tensor"), + ("seg_label", "segmentation"), + ("clust_label", "cluster_label"), + ("particles", "particle"), + ("coord_label", "particle_coords"), + ("ppn_label", "particle_points"), + ("meta", "meta"), + ("graph_label", "particle_graph"), + ("event_info", "event_info"), + ) +} + + +@pytest.fixture +def raw(): + """Pixel charge contributions with overlap, LE, an unknown hit and ancestry.""" + table = dict( + parent=np.array([0, 0, 0]), + ancestor=np.array([0, 0, 0]), + fragment=np.arange(3), + instance=np.array([0, 1, -1]), + fragment_instance=np.array([0, 1, 0]), + interaction=np.array([0, 0, 1]), + pdg=np.array([13, 22, 11]), + parent_pdg=np.array([13, 13, 13]), + geant4_trackid=np.array([10, 80, 150]), + geant4_parent_trackid=np.array([-1, 10, 10]), + geant4_parent_pdg=np.array([0, 13, 13]), + motif=np.array([1, 0, 4]), + instance_sem_type=np.array([2, 1, -1]), + is_primary=np.array([1, 0, 0]), + ) + for key in ("start_pos", "end_pos", "first_pos", "last_pos"): + table[key] = np.array( + [[0.5, 0.5, 0.5, 10.0], [1.5, 0.5, 0.5, 20.0], [2.5, 0.5, 0.5, 30.0]] + ) + table["last_pos"][0, 0] = 2.5 + for key in ("start_mom", "end_mom"): + table[key] = np.tile([0.0, 0.0, 100.0, 150.0], (3, 1)) + geometry = dict( + x_anode_mm=0.0, + drift_direction=-1, + reference_tick=0.0, + mm_per_tick=1.0, + y_min_mm=0.0, + z_min_mm=0.0, + pitch_mm=1.0, + ) + return { + "pimm": dict( + hits=dict( + coord=np.array( + [ + [0.0, 0.0, 0.5], + [0.0, 0.0, 1.5], + [0.0, 0.0, 2.5], + [0.0, 0.0, 0.5], + [0.0, 0.0, 3.5], + ] + ), + energy=np.array([[2.0], [3.0], [4.0], [9.0], [-1.0]]), + plane_id=np.zeros((5, 1), int), + planes=["volume_0_Pixel"], + readout_type="pixel", + segment_motif=np.array([[1], [0], [4], [0], [-1]]), + instance_fragment=np.array([[0], [1], [2], [1], [-1]]), + instance_particle=np.array([[0], [1], [0], [1], [-1]]), + ), + particle=table, + interaction={"vertex": np.array([[10.0, 20.0, 30.0], [99.0, 99.0, 99.0]])}, + metadata=dict( + pixel_geometry=[geometry], hit_x_from="nominal", hit_energy="charge" + ), + identity={"event_id": 12345, "entry": 0}, + ) + } + + +def round_trip_cache(batch, products, path, stage): + """Publish native products and reload them through cache collation. + + Parameters + ---------- + batch : dict + Collated input batch containing physical-source provenance. + products : dict + Batched products to cache, without source metadata. + path : pathlib.Path + Temporary cache repository. + stage : str + Cache stage name. + + Returns + ------- + dict + Re-collated products from the published cache stage. + """ + from spine.io.collate import CollateAll + from spine.io.read import CacheReader + from spine.io.unwrap import Unwrapper + from spine.io.write import CacheWriter + + metadata = { + key: value + for key, value in batch.items() + if key == "index" or key.startswith("source_file_") + } + payload = { + key: value.to_numpy() if hasattr(value, "to_numpy") else value + for key, value in products.items() + } + writer = CacheWriter(path=str(path), stage=stage, keys=list(products)) + writer(Unwrapper()({**metadata, **payload}), {}) + writer.finalize() + writer.close() + reader = CacheReader(path=str(path), stage=stage) + events = reader.get_many(list(range(len(batch["index"])))) + reader.close() + return CollateAll(data_keys=products)(events) diff --git a/test/test_io/test_pimm/coverage.ini b/test/test_io/test_pimm/coverage.ini new file mode 100644 index 00000000..c36b93cf --- /dev/null +++ b/test/test_io/test_pimm/coverage.ini @@ -0,0 +1,13 @@ +# Strict coverage for the PIMM adapter, including error paths. +[run] +branch = True +source = src/spine + +[report] +include = + src/spine/io/read/pimm.py + src/spine/io/dataset/pimm.py + src/spine/io/parse/pimm/* +exclude_lines = +fail_under = 100 +show_missing = True diff --git a/test/test_io/test_pimm/test_parsers.py b/test/test_io/test_pimm/test_parsers.py new file mode 100644 index 00000000..4f9947e0 --- /dev/null +++ b/test/test_io/test_pimm/test_parsers.py @@ -0,0 +1,211 @@ +"""PIMM charge/label alignment, truth and malformed-input regression tests.""" + +import copy + +import numpy as np +import pytest + +from spine.io.parse.pimm import ( + PIMMClusterLabelParser, + PIMMEventInfoParser, + PIMMMetaParser, + PIMMParticleCoordsParser, + PIMMParticleGraphParser, + PIMMParticleParser, + PIMMParticlePointsParser, + PIMMSegmentationParser, + PIMMTensorParser, +) + + +def test_aligned_charge_and_truth(raw): + """Charge overlaps share winners; unknown inputs keep an ignored target.""" + parser = PIMMTensorParser("float32", voxel_size_mm=1, include_unlabeled=True) + data = parser(raw) + seg = PIMMSegmentationParser("float32", voxel_size_mm=1, include_unlabeled=True)( + raw + ) + clust = PIMMClusterLabelParser("float32", voxel_size_mm=1, include_unlabeled=True)( + raw + ) + np.testing.assert_array_equal(data.features[:, 0], [11, 3, 4, -1]) + np.testing.assert_array_equal(seg.features[:, 0], [1, 0, 4, -1]) + np.testing.assert_array_equal(data.coords, clust.coords) + np.testing.assert_array_equal(data.coords, seg.coords) + np.testing.assert_array_equal(clust.particles["particle"], [0, 1, 2]) + np.testing.assert_array_equal(clust.particles["interaction"], [0, 0, 0]) + np.testing.assert_array_equal(clust.particles["pid"], [2, 0, 2]) + assert ( + PIMMMetaParser("float32", voxel_size_mm=1, include_unlabeled=True)(raw) + is data.meta + ) + assert len(raw["_pimm_prepared"]) == 1 + particles = PIMMParticleParser("float32", voxel_size_mm=1, include_unlabeled=True)( + raw + ) + assert all(np.isnan(p.energy_deposit) for p in particles) + assert particles[0].parent_track_id == -1 + assert particles[1].pid == 0 + assert particles[0].interaction_primary == 1 + np.testing.assert_array_equal(particles[0].children_id, [1, 2]) + np.testing.assert_array_equal( + PIMMParticleGraphParser("float32")(raw).index, [[0, 0], [1, 2]] + ) + identity = PIMMEventInfoParser("float32")(raw) + assert identity.features.shape == (1, 2) + assert 12345 in identity.features + + +def test_endpoints_and_missing_ancestry(raw): + """Missing endpoints remain unknown; -1 never indexes the final row.""" + table = raw["pimm"]["particle"] + table["first_pos"][2] = np.nan + table["ancestor"][2] = table["parent"][2] = -1 + table["fragment_instance"][2] = -1 + coords = PIMMParticleCoordsParser("float32", voxel_size_mm=1)(raw) + assert np.isnan(coords.coordinate_data[2]).all() + particles = PIMMParticleParser("float32", voxel_size_mm=1)(raw) + assert particles[2].ancestor_track_id == -1 + ppn = PIMMParticlePointsParser("float32", voxel_size_mm=1)(raw) + assert len(ppn.coords) == 3 + assert ppn.features.shape[1] == 3 + assert set(coords.coordinate_groups) == {"start", "end"} + + +def test_empty_event(raw): + """Empty point clouds and tables retain stable dimensions.""" + event = raw["pimm"] + for key, value in event["hits"].items(): + if isinstance(value, np.ndarray): + event["hits"][key] = value[:0] + for key, value in event["particle"].items(): + event["particle"][key] = value[:0] + for parser in ( + PIMMTensorParser, + PIMMSegmentationParser, + PIMMClusterLabelParser, + PIMMParticleCoordsParser, + PIMMParticlePointsParser, + ): + assert len(parser("float32")(raw).features) == 0 + assert len(PIMMParticleParser("float32")(raw)) == 0 + assert PIMMParticleGraphParser("float32")(raw).index.shape == (2, 0) + + +@pytest.mark.parametrize( + "kwargs", + [ + dict(voxel_size_mm=0), + dict(voxel_size_mm=np.nan), + dict(origin_mm=(0, 0)), + dict(origin_mm=(0, np.inf, 0)), + ], +) +def test_invalid_grid(kwargs): + """Reject grids that cannot define a detector-to-voxel transform.""" + with pytest.raises(ValueError): + PIMMTensorParser("float32", **kwargs) + + +@pytest.mark.parametrize( + "part,key,value", + [ + ("hits", "readout_type", "wire"), + ("metadata", "pixel_geometry", None), + ("metadata", "hit_x_from", "true_t0"), + ("metadata", "hit_energy", "energy"), + ("hits", "energy", np.array([[np.nan]] * 5)), + ("hits", "instance_fragment", np.array([[99]] * 5)), + ("hits", "segment_motif", np.array([[8]] * 5)), + ("hits", "energy", np.array([[1.0]])), + ], +) +def test_invalid_event(raw, part, key, value): + """Reject unsupported geometry, corrupt references and misaligned arrays.""" + raw["pimm"][part][key] = value + with pytest.raises(ValueError): + PIMMTensorParser("float32")(raw) + + +def test_batch_and_truth_round_trip(raw, tmp_path): + """Particle references shift per event and physical truth survives HDF5.""" + from spine.config.factory import instantiate + from spine.construct import BuildManager + from spine.io.collate import CollateAll + from spine.io.dataset import PIMMDataset + from spine.io.read import HDF5Reader + from spine.io.unwrap import Unwrapper + from spine.io.write import HDF5Writer + + from .conftest import FULL_SCHEMA + + events = [] + for index in range(2): + trees = copy.deepcopy(raw) + event = { + key: instantiate( + PIMMDataset.parser_dict, + dict(cfg, voxel_size_mm=1) if key != "event_info" else cfg, + dtype="float32", + )(trees) + for key, cfg in FULL_SCHEMA.items() + } + # Unlabelled hits do not contribute to truth objects. + event["index"] = index + events.append(event) + batch = CollateAll(data_keys=events[0])(events) + restored = Unwrapper()(batch) + BuildManager(True, True, True, mode="truth", units="cm")(restored) + assert len(restored["truth_particles"][0]) > 0 + assert restored["truth_particles"][0][0].units == "cm" + path = tmp_path / "truth.h5" + writer = HDF5Writer( + file_name=str(path), + keys=list(FULL_SCHEMA) + + ["truth_fragments", "truth_particles", "truth_interactions"], + format_version=2, + ) + writer(restored, {}) + writer.finalize() + writer.close() + reloaded = HDF5Reader(file_keys=str(path))[0] + assert len(reloaded["truth_particles"]) == len(restored["truth_particles"][0]) + + +def test_unavailable_truth_and_ppn_rejection(raw): + """Unknown vertices and unsupported or missing endpoints do not become targets.""" + event = raw["pimm"] + event["interaction"] = {} + table = event["particle"] + table.pop("geant4_parent_trackid") + table["instance_sem_type"][0] = 5 + table["first_pos"][1] = np.nan + assert len(PIMMParticlePointsParser("float32")(raw).features) == 0 + assert all( + np.isnan(p.ancestor_position).all() for p in PIMMParticleParser("float32")(raw) + ) + + +def test_opposite_drift_volumes(raw): + """Different anode faces map identical ticks to opposite physical positions.""" + geometry = raw["pimm"]["metadata"]["pixel_geometry"] + geometry.append( + dict(geometry[0], x_anode_mm=10.0, drift_direction=1, reference_tick=1.0) + ) + hits = raw["pimm"]["hits"] + hits["planes"].append("volume_1_Pixel") + hits["plane_id"][1] = 1 + positions = PIMMTensorParser._coordinates(hits, raw["pimm"]["metadata"]) + np.testing.assert_allclose(positions[0], [0.5, 0.5, 0.5]) + np.testing.assert_allclose(positions[1], [9.5, 0.5, 0.5]) + + +def test_unlabelled_voxels_excluded_for_training(raw): + """Default training retains charge from all contributors in labelled voxels.""" + tensor = PIMMTensorParser("float32", voxel_size_mm=1)(raw) + labels = PIMMSegmentationParser("float32", voxel_size_mm=1)(raw) + np.testing.assert_array_equal(tensor.features[:, 0], [11, 3, 4]) + assert np.all(labels.features >= 0) + raw["pimm"]["hits"]["segment_motif"][:] = -1 + raw.pop("_pimm_prepared") + assert len(PIMMTensorParser("float32")(raw).features) == 0 diff --git a/test/test_io/test_pimm/test_pimm_chain.py b/test/test_io/test_pimm/test_pimm_chain.py new file mode 100644 index 00000000..736058c4 --- /dev/null +++ b/test/test_io/test_pimm/test_pimm_chain.py @@ -0,0 +1,152 @@ +"""Opt-in real-sample downstream training and truth conversion integration.""" + +import os +from copy import deepcopy +from pathlib import Path + +import numpy as np +import pytest + +from spine.io.collate import CollateAll +from spine.io.dataset import PIMMDataset + +from .conftest import FULL_SCHEMA, round_trip_cache +from .truth_helpers import check_truth_round_trip + +SAMPLE = os.environ.get("SPINE_PIMM_ROOT") +pytestmark = pytest.mark.skipif(not SAMPLE, reason="Set SPINE_PIMM_ROOT") + + +def test_real_truth_conversion(tmp_path): + """Build two complete real events and preserve cm geometry and associations.""" + ds = PIMMDataset( + data_root=SAMPLE, + dataset_name="sim_pixel", + schema=FULL_SCHEMA, + dtype="float32", + n_entry=2, + ) + events = [ds[0], ds[1]] + truth = check_truth_round_trip(events, tmp_path / "truth.h5") + for index, event in enumerate(events): + assert all(np.isnan(p.energy_deposit) for p in event["particles"]) + assert len(truth["truth_fragments"][index]) > 0 + assert len(truth["truth_particles"][index]) > 0 + assert any(len(p.children_id) for p in truth["truth_particles"][index]) + + +@pytest.mark.parametrize("path", ["shower", "track", "inter"]) +def test_real_grappa_targets_and_backward(path, tmp_path): + """Every generic GrapPA objective has eligible targets and nonzero gradients.""" + torch = pytest.importorskip("torch") + pytest.importorskip("torch_geometric") + from spine.config import load_config_file + from spine.data import TensorBatch + from spine.model.grappa import GrapPA, GrapPALoss + + torch.manual_seed(7) + ds = PIMMDataset( + data_root=SAMPLE, + dataset_name="sim_pixel", + schema=FULL_SCHEMA, + dtype="float32", + n_entry=2, + ) + # Two events exercise the coordinate-table offsets used by orientation targets. + batch = CollateAll(data_keys=ds.data_keys)([ds[0], ds[1]]) + labels = batch["clust_label"].to_tensor(dtype=torch.float32) + coords = batch["coord_label"].to_tensor(dtype=torch.float32) + config_path = ( + Path(__file__).resolve().parents[3] + / "config/full_chain/full_chain_regression.yaml" + ) + config = load_config_file(str(config_path), download=False)["model"]["modules"] + model_config = deepcopy(config["grappa_" + path]) + loss_config = deepcopy(config["grappa_" + path + "_loss"]) + # Keep every objective and geometry feature; shrink only the neural hidden width. + gnn = model_config["gnn_model"] + gnn["edge_layer"]["mlp"]["width"] = 8 + for name in ("message_mlp", "aggr_mlp"): + gnn["node_layer"][name]["width"] = 8 + if path == "inter": + loss_config["node_loss"]["primary"]["use_closest"] = True + loss_config["return_targets"] = True + model = GrapPA(model_config).train() + objective = GrapPALoss(loss_config, model_config) + + graph = model.materialize_graph(data=labels, coord_label=coords) + targets = objective.materialize_targets( + clust_label=labels, coord_label=coords, **graph + ) + for name, valid in targets.items(): + if name.endswith("_valid"): + mask = valid.numpy_tensor().astype(bool) + assert mask.sum() > 0, (path, name) + values = targets[name.removesuffix("_valid") + "_target"].numpy_tensor() + assert np.all(values[mask] >= 0), (path, name) + + # Publish fragment/particle graph targets with their real source identities. + graph_keys = ( + "clusts", + "edge_index", + "node_features", + "edge_features", + "shapes", + "start_points", + "end_points", + ) + cached = round_trip_cache( + batch, + {**{key: graph[key] for key in graph_keys if key in graph}, **targets}, + tmp_path / "chain.spine-cache", + "particle_graph" if path == "inter" else "fragment_graph", + ) + cached = { + key: ( + value.to_tensor(dtype=torch.float32) + if isinstance(value, TensorBatch) + else value + ) + for key, value in cached.items() + } + for name in targets: + np.testing.assert_array_equal( + cached[name].to_numpy().numpy_tensor(), targets[name].numpy_tensor() + ) + + # Train from the graph cache without replacing or dropping any objective. + output = model( + **{ + key: cached[key] + for key in ( + "clusts", + "edge_index", + "node_features", + "edge_features", + "shapes", + ) + if key in cached + } + ) + output = {**cached, **output} + metrics = objective(clust_label=labels, coord_label=coords, **output) + report = {} + for name, loss in metrics.items(): + if name.endswith("_loss"): + prefix = name.removesuffix("_loss") + assert metrics[prefix + "_count"] > 0, (path, prefix) + assert torch.isfinite(loss), (path, prefix) + gradients = torch.autograd.grad( + loss, tuple(model.parameters()), retain_graph=True, allow_unused=True + ) + assert all(torch.isfinite(g).all() for g in gradients if g is not None) + assert any(g is not None and g.abs().sum() > 0 for g in gradients), ( + path, + prefix, + ) + report[prefix] = { + "count": metrics[prefix + "_count"], + "loss": float(loss.detach()), + } + metrics["loss"].backward() + print({"path": path, "objectives": report}) diff --git a/test/test_io/test_pimm/test_pimm_model.py b/test/test_io/test_pimm/test_pimm_model.py new file mode 100644 index 00000000..ce32f713 --- /dev/null +++ b/test/test_io/test_pimm/test_pimm_model.py @@ -0,0 +1,151 @@ +"""Opt-in real-file training smoke test using models available on main. + +Run with SPINE_PIMM_ROOT pointing at paired hits/labl directories. +""" + +import os + +import numpy as np +import pytest + +from .conftest import round_trip_cache + +SAMPLE = os.environ.get("SPINE_PIMM_ROOT") +pytestmark = pytest.mark.skipif( + not SAMPLE, reason="Set SPINE_PIMM_ROOT for real-file training" +) + + +def test_real_pimm_forward_backward(tmp_path): + """Backpropagate every training objective through the complete real event.""" + torch = pytest.importorskip("torch") + pytest.importorskip("MinkowskiEngine") + pytest.importorskip("torch_cluster") + from spine.io.collate import CollateAll + from spine.io.dataset import PIMMDataset + from spine.model.full_chain.providers.segmentation import SegmentationStage + from spine.model.full_chain.state import ChainState + from spine.model.graph_spice import GraphSPICE, GraphSPICELoss + from spine.model.uresnet.ppn import UResNetPPN, UResNetPPNLoss + + torch.manual_seed(7) + schema = { + key: {"provider": provider} + for key, provider in ( + ("data", "tensor"), + ("seg_label", "segmentation"), + ("clust_label", "cluster_label"), + ("ppn_label", "particle_points"), + ) + } + ds = PIMMDataset( + data_root=SAMPLE, + dataset_name="sim_pixel", + dtype="float32", + schema=schema, + n_entry=1, + ) + e = ds[0] + np.testing.assert_array_equal( + e["data"].coordinate_data, e["seg_label"].coordinate_data + ) + np.testing.assert_array_equal(e["data"].coordinate_data, e["clust_label"].coords) + np.testing.assert_array_equal( + e["seg_label"].features[:, 0], e["clust_label"].shapes + ) + # Keep the full real event, including all overlaps and endpoint targets. + source_batch = CollateAll(data_keys=ds.data_keys)([e]) + batch = CollateAll(data_keys=schema)([e]) + batch = {k: v.to_tensor(dtype=torch.float32) for k, v in batch.items()} + size = int(e["data"].meta.count.max()) + 2 + backbone = { + "num_classes": 5, + "num_input": 1, + "filters": 4, + "depth": 2, + "reps": 1, + "spatial_size": size, + "activation": "relu", + "norm_layer": "none", + } + + # Segmentation and PPN share the standard UResNet backbone on main. + model = UResNetPPN(backbone, ppn={}).train() + objective = UResNetPPNLoss( + backbone, uresnet_loss={}, ppn={}, ppn_loss={"return_mask_labels": True} + ) + stage = SegmentationStage("segmentation", "uresnet", model, None) + output = stage(ChainState(**batch)).outputs + metrics = objective(batch["seg_label"], batch["ppn_label"], **output) + cached = round_trip_cache( + source_batch, + {"seg_pred": output["seg_pred"], "ppn_points": output["ppn_points"]}, + tmp_path / "segmentation.spine-cache", + "segmentation", + ) + for name in cached: + np.testing.assert_allclose( + cached[name].to_numpy().numpy_tensor(), + output[name].to_numpy().numpy_tensor(), + ) + assert len(e["seg_label"].features) > 0 + assert len(e["ppn_label"].features) > 0 + positive_points = int(metrics["ppn_mask_labels"][0].torch_tensor().sum()) + assert positive_points > 0 + assert metrics["ppn_type_loss"] > 0 and metrics["ppn_reg_loss"] > 0 + parameter = next(model.uresnet.backbone.parameters()) + report = {} + for name in ("uresnet_loss", "ppn_loss"): + assert torch.isfinite(metrics[name]) + grad = torch.autograd.grad(metrics[name], parameter, retain_graph=True)[0] + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + report[name] = float(metrics[name].detach()) + + # Graph-SPICE uses the same aligned voxel labels with its own mainline backbone. + graph_config = { + "use_raw_features": True, + "embedder": { + "coord_conv": False, + "uresnet": { + key: value for key, value in backbone.items() if key != "num_classes" + }, + }, + "kernel": {"provider": "bilinear", "num_features": 4}, + "constructor": { + "graph": {"provider": "radius", "r": 1.9}, + "min_size": 1, + "edge_threshold": 0.5, + }, + } + graph = GraphSPICE(graph_config).train() + graph_objective = GraphSPICELoss(graph_config, {"provider": "edge", "metric": None}) + graph_metrics = graph_objective( + batch["seg_label"], + batch["clust_label"], + **graph(batch["data"], batch["seg_label"]), + ) + assert graph_metrics["count"] > 0 + assert torch.isfinite(graph_metrics["loss"]) + graph_parameter = next(graph.embedder.backbone.parameters()) + grad = torch.autograd.grad( + graph_metrics["loss"], graph_parameter, retain_graph=True + )[0] + assert torch.isfinite(grad).all() and grad.abs().sum() > 0 + report["graph_spice_loss"] = float(graph_metrics["loss"].detach()) + + total = metrics["loss"] + graph_metrics["loss"] + total.backward() + assert any( + p.grad is not None and p.grad.abs().sum() > 0 + for p in graph.kernel_fn.parameters() + ) + print( + { + "voxels": len(e["data"].features), + "ppn_targets": len(e["ppn_label"].features), + "ppn_positive_voxels": positive_points, + "graph_spice_edges": graph_metrics["count"], + "losses": report, + "total_loss": float(total.detach()), + } + ) diff --git a/test/test_io/test_pimm/test_reader.py b/test/test_io/test_pimm/test_reader.py new file mode 100644 index 00000000..526d04f2 --- /dev/null +++ b/test/test_io/test_pimm/test_reader.py @@ -0,0 +1,159 @@ +"""Exercise the real pimm-data backend, including worker serialization.""" + +import json +import pickle + +import h5py +import numpy as np +import pytest + +from spine.io.dataset import PIMMDataset +from spine.io.read import PIMMReader + +from .conftest import FULL_SCHEMA + + +@pytest.fixture +def corpus(tmp_path, raw): + """Use upstream's synthetic writer instead of mocking its label joins.""" + pytest.importorskip("pimm_data") + from pimm_data.testing import make_jaxtpc_sample + + root = str(tmp_path / "corpus") + make_jaxtpc_sample(root, readout_type="pixel", n_events=2, n_files=2) + for path in (tmp_path / "corpus" / "labl").glob("*.h5"): + with h5py.File(path, "r+") as file: + meta = raw["pimm"]["metadata"] + file["config"].attrs["supera_pixel_geometry"] = json.dumps( + meta["pixel_geometry"] * 2 + ) + file["config"].attrs["supera_hit_x_from"] = "nominal" + file["config"].attrs["supera_hit_energy"] = "charge" + return root + + +def test_reader_selection_pickle_and_labels(corpus): + """Selection crosses files without changing event or particle identity.""" + reader = PIMMReader(data_root=corpus, dataset_name="sim", entry_list=[3, 0, 2]) + first = reader[0] + assert first["file_index"] == 1 + assert first["file_entry_index"] == 1 + assert first["source_file_entry_index"] == 1 + assert ( + first["pimm"]["identity"]["event_id"] + != reader[1]["pimm"]["identity"]["event_id"] + ) + restored = pickle.loads(pickle.dumps(reader)) + np.testing.assert_array_equal( + first["pimm"]["hits"]["energy"], restored[0]["pimm"]["hits"]["energy"] + ) + reader.close() + restored.close() + + +def test_dataset_factories_and_grid(corpus): + """The public pimm provider parses the combined event through common machinery.""" + from spine.io.factories import dataset_factory, reader_factory + + cfg = dict(provider="pimm", data_root=corpus, dataset_name="sim") + reader = reader_factory(cfg) + assert isinstance(reader, PIMMReader) + reader.close() + ds = dataset_factory(dict(cfg, schema=FULL_SCHEMA), dtype="float32") + event = ds[0] + assert len(event["data"].features) + np.testing.assert_array_equal(event["data"].coords, event["seg_label"].coords) + assert np.isnan(event["particles"][0].energy_deposit) + assert set(ds[0]) == set(ds.__getitems__([0])[0]) + ds.reader.close() + with pytest.raises(ValueError, match="same voxel"): + PIMMDataset( + data_root=corpus, + schema={ + "data": dict(provider="tensor", voxel_size_mm=1), + "seg_label": dict(provider="segmentation", voxel_size_mm=3), + }, + dtype="float32", + ) + + +def test_invalid_projection(): + """Reject legacy raw products rather than silently reading truth deposits.""" + with pytest.raises(ValueError, match="aligned"): + PIMMReader(keys=["points"]) + + +def test_missing_dependency(monkeypatch): + """Base SPINE imports stay lightweight and missing extras fail clearly.""" + from spine.io.read import pimm + + def missing(name): + raise ImportError(name) + + monkeypatch.setattr(pimm, "import_module", missing) + with pytest.raises(ImportError, match=r"spine\[pimm\]"): + PIMMReader() + + +def test_missing_optional_metadata(corpus, monkeypatch, tmp_path): + """Absent event identity or geometry is not replaced with fabricated values.""" + reader = PIMMReader(data_root=corpus, dataset_name="sim") + monkeypatch.setattr(reader.dataset.hits_reader, "event_id_of", lambda _: None) + assert set(reader[0]["pimm"]["identity"]) == {"entry"} + reader.close() + path = tmp_path / "metadata.h5" + with h5py.File(path, "w") as file: + file.create_group("config") + assert PIMMReader._read_metadata(str(path))["pixel_geometry"] is None + + +def test_plane_local_groups(corpus): + """The final-layout per-plane group namespace is resolved by pimm-data.""" + from pathlib import Path + + from pimm_data._shard_meta import clear_cache + + before = PIMMReader(data_root=corpus, dataset_name="sim") + expected = before[0]["pimm"]["hits"] + before.close() + hits_path = next((Path(corpus) / "hits").glob("*.h5")) + labl_path = Path(corpus) / "labl" / hits_path.name.replace("_hits_", "_labl_") + with h5py.File(hits_path, "r+") as hits, h5py.File(labl_path, "r+") as labels: + hits.attrs["group_id_space"] = "plane" + for key in (k for k in hits if k.startswith("entry_")): + for volume in (k for k in hits[key] if k.startswith("volume_")): + plane = hits[f"{key}/{volume}/Pixel"] + old = plane["group_ids"][:] + # Reverse local numbering to make an accidental direct join wrong. + plane["group_ids"][:] = np.arange(len(old))[::-1] + target = labels[f"{key}/{volume}"].require_group("plane_Pixel") + target["group"] = old[::-1] + clear_cache() + after = PIMMReader(data_root=corpus, dataset_name="sim") + actual = after[0]["pimm"]["hits"] + for name in ( + "coord", + "energy", + "instance_fragment", + "instance_particle", + "segment_motif", + ): + np.testing.assert_array_equal(actual[name], expected[name]) + after.close() + + +def test_runs_and_explicit_files(corpus, tmp_path): + """Task discovery and explicit modality paths select the same events.""" + from pathlib import Path + + root = Path(corpus) + files = {key: str(root / key / "*.h5") for key in ("hits", "labl")} + explicit = PIMMReader(files=files, dataset_name="sim", split="") + identity = explicit[0]["pimm"]["identity"] + explicit.close() + task = tmp_path / "runs" / "task" + task.parent.mkdir() + root.rename(task) + runs = PIMMReader(data_root=str(task.parent), runs=["task"], dataset_name="sim") + assert runs[0]["pimm"]["identity"] == identity + runs.close() diff --git a/test/test_io/test_pimm/truth_helpers.py b/test/test_io/test_pimm/truth_helpers.py new file mode 100644 index 00000000..692e25d0 --- /dev/null +++ b/test/test_io/test_pimm/truth_helpers.py @@ -0,0 +1,89 @@ +"""Physical truth round-trip assertions shared by integration tests.""" + +import copy + +import numpy as np + +from spine.construct import BuildManager +from spine.io.collate import CollateAll +from spine.io.read import HDF5Reader +from spine.io.unwrap import Unwrapper +from spine.io.write import HDF5Writer + + +def check_truth_round_trip(events, path): + """Build all truth levels in cm and retain their associations through HDF5. + + Parameters + ---------- + events : list[dict] + Parsed events with the complete native schema and source provenance. + path : pathlib.Path + Temporary output HDF5 file. + + Returns + ------- + dict + Unwrapped truth products used for physical validation by callers. + """ + batch = CollateAll(data_keys=events[0])(events) + truth = Unwrapper()(batch) + original = copy.deepcopy(truth) + BuildManager(True, True, True, mode="truth", units="cm")(truth) + keys = [ + "meta", + "event_info", + "points_label", + "depositions_label", + "truth_fragments", + "truth_particles", + "truth_interactions", + ] + writer = HDF5Writer(file_name=str(path), keys=keys, format_version=2) + writer(truth, {}) + writer.finalize() + writer.close() + reader = HDF5Reader(file_keys=str(path)) + for index, event in enumerate(events): + restored = reader[index] + BuildManager(True, True, True, mode="truth", units="cm")(restored) + # BuildManager converts parsed truth to cm; voxel centers use +0.5. + expected = event["meta"].to_cm( + original["clust_label"][index].coords, center=True + ) + np.testing.assert_allclose(restored["points_label"], expected) + assert np.isclose( + restored["depositions_label"].sum(), event["data"].features.sum() + ) + for key in ("truth_fragments", "truth_particles", "truth_interactions"): + assert len(restored[key]) == len(truth[key][index]) + for before, after in zip(truth[key][index], restored[key]): + assert after.units == "cm" + assert before.id == after.id + np.testing.assert_allclose(before.points, after.points) + for field in ( + "parent_id", + "children_id", + "orig_id", + "pid", + "is_primary", + "vertex", + "energy_deposit", + "fragment_ids", + "particle_ids", + ): + if hasattr(before, field): + np.testing.assert_allclose( + getattr(before, field), getattr(after, field) + ) + for particle in restored["truth_particles"]: + source = event["particles"][particle.orig_id] + assert particle.pid == source.pid + assert particle.is_primary == (source.interaction_primary > 0) + for particle in truth["truth_particles"][index]: + for fragment in particle.fragments: + assert fragment.particle_id == particle.id + for interaction in truth["truth_interactions"][index]: + for particle in interaction.particles: + assert particle.interaction_id == interaction.id + return truth