Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 205 additions & 8 deletions sotrplib/sources/stacking.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from abc import ABC
from abc import ABC, abstractmethod

import numpy as np
import structlog
from astropy import units as u
from pixell import enmap, reproject
from structlog.types import FilteringBoundLogger

from sotrplib.maps.core import (
ProcessableMap,
Expand Down Expand Up @@ -39,7 +41,149 @@ def __init__(
self.times = times


class Stacker(ABC):
class StackedStamp:
"""
Class to hold a stacked flux stamp and its corresponding ivar stamp.
This is essentially just a single stamp but for clarity we define a separate class for it.
Also unlike a stamp it does not have a time associated with it, since it is a combination of multiple stamps.

Attributes:
-----------
flux: enmap.ndmap
The stacked flux stamp.
ivar: enmap.ndmap
The corresponding ivar stamp for the stacked flux.
"""

def __init__(self, flux: enmap.ndmap, ivar: enmap.ndmap):
self.flux = flux
self.ivar = ivar


def clean_stamps(stamps: Stamps, log: FilteringBoundLogger | None = None) -> Stamps:
"""
Clean the stamps by removing any stamps with non-finite times or any
points with zero/NaN ivar. Also removes any stamps with non-finite flux.

Parameters:
-----------
stamps: Stamps
The Stamps object to clean.
log: FilteringBoundLogger | None
Optional logger for logging information. If not provided, a default logger will be used.

Returns:
--------
Stamps
A new Stamps object with only the usable stamps.
"""
log = (log or structlog.get_logger()).bind(func="clean_stamps")
n_bad_time = 0
n_bad_pixels = 0
cleaned_flux_stamps = []
cleaned_ivar_stamps = []
cleaned_times = []

for flux_stamp, ivar_stamp, time in zip(
stamps.flux_stamps, stamps.ivar_stamps, stamps.times
):
if not np.isfinite(time):
n_bad_time += 1
continue
if not (
np.all(np.isfinite(flux_stamp))
and np.all(np.isfinite(ivar_stamp))
and np.all(ivar_stamp > 0)
):
n_bad_pixels += 1
continue

cleaned_flux_stamps.append(flux_stamp)
cleaned_ivar_stamps.append(ivar_stamp)
cleaned_times.append(time)

n_input = len(stamps.flux_stamps)
n_kept = len(cleaned_flux_stamps)

log.debug(
"clean_stamps.summary",
n_input=n_input,
n_kept=n_kept,
n_bad_time=n_bad_time,
n_bad_pixels=n_bad_pixels,
)

if n_kept == 0 and n_input > 0:
log.warning("clean_stamps.no_usable_stamps", n_input=n_input)
return Stamps(
flux_stamps=cleaned_flux_stamps,
ivar_stamps=cleaned_ivar_stamps,
times=cleaned_times,
)


class StackingMethod(ABC):
@abstractmethod
def combine(
self, flux: np.ndarray, ivar: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
"""flux, ivar shaped (N, ny, nx) -> stacked (ny, nx) flux and ivar."""

def stack(
self, stamps: Stamps, log: FilteringBoundLogger | None = None
) -> list[StackedStamp]:
"""
Stack the stamps. Combine controls the actual stacking method,
and the stack function handles the cleaning of the stamps and the creation of the StackedStamp object.
In general this function should not be overridden, but rather the combine function should be overridden to implement a new stacking method.

Parameters:
-----------
stamps: Stamps
The Stamps object containing the flux and ivar stamps to be stacked.
log: FilteringBoundLogger | None
Optional logger for logging information. If not provided, a default logger will be used.

Returns:
--------
list[StackedStamp]
A list containing a single StackedStamp object with the stacked flux and ivar stamps.
"""
log = (log or structlog.get_logger()).bind(method=type(self).__name__)
stamps = clean_stamps(stamps=stamps, log=log)
if not stamps.flux_stamps:
return []
wcs = stamps.flux_stamps[0].wcs
flux = np.stack(stamps.flux_stamps)
ivar = np.stack(stamps.ivar_stamps)
f, w = self.combine(flux, ivar)
log.debug("StackingMethod.stack.stacked", n_stamps=len(stamps.flux_stamps))
return [StackedStamp(enmap.enmap(f, wcs), enmap.enmap(w, wcs))]


class MeanStacking(StackingMethod):
def combine(
self, flux: np.ndarray, ivar: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
# Compute the simple mean stack.
# Return (mean_flux, mean_ivar).
mean_flux = np.mean(flux, axis=0)
mean_ivar = len(ivar) ** 2 / np.sum(1 / ivar, axis=0)
return mean_flux, mean_ivar


class InverseVarianceStacking(StackingMethod):
def combine(
self, flux: np.ndarray, ivar: np.ndarray
) -> tuple[np.ndarray, np.ndarray]:
# Compute the inverse variance weighted mean stack.
# Return (weighted_mean_flux, weighted_mean_ivar).
weighted_mean_flux = np.sum(flux * ivar, axis=0) / np.sum(ivar, axis=0)
weighted_mean_ivar = np.sum(ivar, axis=0)
return weighted_mean_flux, weighted_mean_ivar


class Stacker:
"""
Stacking class to handle the the creation of flux and ivar stamps for a list of sources.

Expand All @@ -50,25 +194,38 @@ class Stacker(ABC):
preprocessors: list[MapPreprocessor]
List of preprocessors to apply to the maps after reading and before stamping.
maps: list[ProcessableMap]
List of maps read from the database and preprocessed, ready for stamping. Starts as none and set by the _read_maps method.
List of maps read from the database and preprocessed, ready for stamping.

Methods:
--------
_read_maps():
_read_map():
Reads maps from the database using the reader, applies preprocessors, and finalizes the maps for stamping.
stamp(sources: list[RegisteredSource], thumb_width=20*u.arcmin) -> Stamps:
stamp(sources: list[RegisteredSource], thumb_width=20*u.arcmin) -> list[Stamps]:
Creates flux and ivar stamps for a list of sources.
stack(sources: list[RegisteredSource], method: StackingMethod, thumb_width=20*u.arcmin) -> list[list[StackedStamp]]:
Stacks the flux and ivar stamps for a list of sources using the specified stacking method
"""

def __init__(
self,
reader: MapCatDatabaseReader, # TODO: Support readers other than MapCatDatabaseReader
preprocessors: list[MapPreprocessor] | None = None,
log: FilteringBoundLogger | None = None,
):
self.reader = reader
self.preprocessors = preprocessors or []
self.maps = self.reader.map_list()

self.stamps: list[Stamps] | None = None # Will be set by the stamp() method
self._stamped_sources: list[RegisteredSource] | None = (
None # Will be set by the stamp() method
)
self._stamped_thumb_width: u.Quantity | None = (
None # Will be set by the stamp() method
)

self.log = log or structlog.get_logger()

def _read_map(self, imap: ProcessableMap) -> ProcessableMap:
imap.build()

Expand All @@ -83,10 +240,17 @@ def _read_map(self, imap: ProcessableMap) -> ProcessableMap:

def stamp(
self, sources: list[RegisteredSource], thumb_width=20 * u.arcmin
) -> Stamps:
) -> list[Stamps]:
log = self.log.bind(func="Stacker.stamp")
log.info(
"Stacker.stamp.start",
n_sources=len(sources),
n_maps=len(self.maps),
thumb_width=thumb_width,
)
stamps = np.empty((len(sources), 3, len(self.maps)), dtype=object)
for j, cur_map in enumerate(self.maps):
cur_map = self._read_map(cur_map)
for j, raw_map in enumerate(self.maps):
cur_map = self._read_map(raw_map)
for i, source in enumerate(sources):
fstamp = reproject.thumbnails(
cur_map.flux,
Expand Down Expand Up @@ -117,6 +281,10 @@ def stamp(
stamps[i, 1, j] = (snrstamp / fstamp) ** 2
stamps[i, 2, j] = time

log.debug("Stacker.stamp.map_done", map_index=j)
cur_map.release()
raw_map.release()

stamps_list = []

for i in range(len(stamps)):
Expand All @@ -128,4 +296,33 @@ def stamp(
)
stamps_list.append(stamp)

self.stamps = stamps_list
self._stamped_sources = list(sources)
self._stamped_thumb_width = thumb_width
log.info("Stacker.stamp.completed")

return stamps_list

def stack(
self,
sources: list[RegisteredSource],
method: StackingMethod,
thumb_width=20 * u.arcmin,
) -> list[list[StackedStamp]]:
log = self.log.bind(func="Stacker.stack", method=type(method).__name__)
cached = (
self.stamps is not None
and self._stamped_sources == sources
and self._stamped_thumb_width == thumb_width
)
log.debug(
"Stacker.stack.using_cached_stamps"
if cached
else "Stacker.stack.restamping"
)
stamps = self.stamps if cached else self.stamp(sources, thumb_width)

return [
method.stack(s, log=log.bind(source_id=src.source_id))
for src, s in zip(sources, stamps)
]
29 changes: 14 additions & 15 deletions tests/test_forced_photometry/test_stacking.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,17 @@
from mapcat.toolkit import update_sky_coverage

from sotrplib.maps.database import FluxMapReader, RhoKappaMapReader
from sotrplib.sources.stacking import Stacker
from sotrplib.sources.stacking import InverseVarianceStacking, Stacker


def check_stacked_peaks(stacked):
"""Each test source has unit flux, so each stack should peak near 1."""
assert len(stacked) == 3
for per_source in stacked:
assert len(per_source) == 1, "every stamp of a source was dropped"
(stack,) = per_source
assert np.all(stack.ivar > 0)
assert 0.95 <= np.amax(stack.flux) <= 1.1


def test_stacker(database_sessionmaker, create_test_maps):
Expand All @@ -29,13 +39,7 @@ def test_stacker(database_sessionmaker, create_test_maps):
)

stacker = Stacker(reader=db_reader)
Stamps = stacker.stamp(sources)
for i in range(3):
simple_stack = np.sum(
Stamps[i].flux_stamps / np.square(Stamps[i].ivar_stamps), axis=0
) / np.sum(1 / np.square(Stamps[i].ivar_stamps), axis=0)
assert np.amax(simple_stack) <= 1.1
assert np.amax(simple_stack) >= 0.95
check_stacked_peaks(stacker.stack(sources, InverseVarianceStacking()))

db_reader = FluxMapReader(
number_to_read=None,
Expand All @@ -48,14 +52,9 @@ def test_stacker(database_sessionmaker, create_test_maps):
)

stacker = Stacker(reader=db_reader)
Stamps = stacker.stamp(sources)
for i in range(3):
simple_stack = np.sum(
Stamps[i].flux_stamps / np.square(Stamps[i].ivar_stamps), axis=0
) / np.sum(1 / np.square(Stamps[i].ivar_stamps), axis=0)
assert np.amax(simple_stack) <= 1.1
assert np.amax(simple_stack) >= 0.95
check_stacked_peaks(stacker.stack(sources, InverseVarianceStacking()))

Stamps = stacker.stamps
for i in range(10):
assert np.isclose(
Stamps[0].times[i] - start_time.unix,
Expand Down
4 changes: 2 additions & 2 deletions tests/test_socat/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ def socat_pickle(tmp_path_factory):
for i in range(128):
client.create_source(
position=ICRS(
ra=random.uniform(0, 360) * u.degree,
dec=random.uniform(-90, 90) * u.degree,
ra=random.uniform(0, 359) * u.degree,
dec=random.uniform(-89, 89) * u.degree,
),
flux=random.uniform(0.01, 1.0) * u.Jy,
name=f"Source {i + 1}",
Expand Down
Loading
Loading