From c405635db654c4e459cc59c863e596b8b046deed Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 13:25:28 -0500 Subject: [PATCH 01/35] feat: initial function for finding tiles from bounding box and tmap --- mapcat/database/tools.py | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 mapcat/database/tools.py diff --git a/mapcat/database/tools.py b/mapcat/database/tools.py new file mode 100644 index 0000000..f372b9a --- /dev/null +++ b/mapcat/database/tools.py @@ -0,0 +1,49 @@ +import numpy as np +from pixell import enmap +from numpy.typing import NDArray + +from .depth_one_map import DepthOneMapTable +from .sky_coverage import SkyCoverageTable + +def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: + """ + Given the bounding box of map, return the list + of sky coverage tiles that cover that map + + Parameters + ---------- + box : NDArray + The bounding box of the map, in the format [[dec_min, ra_max], [dec_max, ra_min]], units in radians + tmap : enmap.enmap + The time map of the depth-one map. Pixels that were observed have non-zero values. + + Returns + ------- + tiles : list + A list of sky coverage tiles that cover the map + """ + + dec_min, ra_max = np.rad2deg(box[0]) + dec_max, ra_min = np.rad2deg(box[1]) + + dec_min = np.floor(dec_min / 10) * 10 + dec_max = np.ceil(dec_max / 10) * 10 + ra_min = np.floor(ra_min / 10) * 10 + ra_max = np.ceil(ra_max / 10) * 10 + + ras = np.arange(ra_min, ra_max, 10) + decs = np.arange(dec_min, dec_max, 10) + + ra_idx = [] + dec_id = [] + + for ra in ras: + for dec in decs: + skybox = np.array([[np.deg2rad(dec), np.deg2rad(ra + 10)], [np.deg2rad(dec + 10), np.deg2rad(ra)]]) + submap = enmap.submap(tmap, skybox) + if np.any(submap): + ra_idx.append(int(ra/10)) + dec_id.append(int(dec/10) + 9) + + return list(zip(ra_idx, dec_id)) + From c141f119741cedd578b8c9208447ad72512ce132 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 13:26:16 -0500 Subject: [PATCH 02/35] feat: update docs and make composite ID --- mapcat/database/sky_coverage.py | 41 ++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index 711b7ac..e4a6f2c 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,34 +2,47 @@ Sky coverage table. """ -from sqlmodel import Field, Relationship, SQLModel +from sqlmodel import Field, Relationship, Index, SQLModel from .depth_one_map import DepthOneMapTable class SkyCoverageTable(SQLModel, table=True): """ - Table for tracking sky coverage for a depth one map. x and y are 0->36 and - 0-18 respectively for CAR patches with 10x10 degrees each. + Table for tracking sky coverage patches with non-zero overlap with a given depth one map. + x and y are 0->36 and 0-18 respectively for CAR patches with 10x10 degrees each. Attributes ---------- - id : str - Internal ID of the sky coverage - map_name : str - Name of depth 1 map being tracked. Foreign into DepthOneMap - patch_coverage : str - String which represents the sky coverage of the d1map + id : uuid.UUID + Unique ID of this coverage instance + map : DepthOneMapTable + Depth 1 map being tracked. Foreign into DepthOneMap + map_id : int + ID of depth 1 map being tracked + x : int + x-index of coverage patch. x=0 runs from RA 0 to 10,, etc. + y : in + y-index of coverage patch. y=0 runs from dec = -90 to -80, etc. """ __tablename__ = "depth_one_sky_coverage" - patch_id: int = Field(primary_key=True) - - x: int = Field(index=True) - y: int = Field(index=True) + x: int = Field(index=True, primary_key=True) + y: int = Field(index=True, primary_key=True) map_id: int = Field( - foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE" + foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE", primary_key=True, ) map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") + + __table_args__ = ( + Index( + "ix_depth_one_sky_coverage_map_id_x_y", + "map_id", + "x", + "y", + unique=True, + pimary_key=True, + ) + ) \ No newline at end of file From 252c0fc8d3d0e3434b146c9a1b91a0faec1c2d10 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 13:28:27 -0500 Subject: [PATCH 03/35] feat: init function to get a list of sky coverage table entries from a d1map --- mapcat/database/tools.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mapcat/database/tools.py b/mapcat/database/tools.py index f372b9a..212bcab 100644 --- a/mapcat/database/tools.py +++ b/mapcat/database/tools.py @@ -47,3 +47,24 @@ def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: return list(zip(ra_idx, dec_id)) +def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: + """ + Get the list of sky coverage tiles that cover a given depth one map + + Parameters + ---------- + d1map : DepthOneMapTable + The depth one map to get the sky coverage for + + Returns + ------- + tiles : list[SkyCoverageTable] + A list of sky coverage tiles that cover the map + """ + + tmap = enmap.read_map(d1map.mean_time_path) + box = d1map.box + + coverage_tiles = get_sky_coverage(box, tmap) + + return [SkyCoverageTable(x=tile[0], y=tile[1], map=d1map, map_id=d1map.map_id) for tile in coverage_tiles] \ No newline at end of file From c0162ed1410712924c65bdefd9fd5ba347690a41 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 13:37:45 -0500 Subject: [PATCH 04/35] fix: ruff formating --- mapcat/database/depth_one_map.py | 3 +++ mapcat/database/sky_coverage.py | 2 +- mapcat/database/tools.py | 3 ++- mapcat/toolkit/act.py | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index 2514636..bc934e3 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -66,6 +66,8 @@ class DepthOneMapTable(SQLModel, table=True): List of pipeline info associed with d1 map depth_one_sky_coverage : list[SkyCoverageTable] List of sky coverage patches for d1 map. + box : list[float] + Bounding box of map in format [[dec_min, ra_max], [dec_max, ra_min]], units in radians. notes: dict[str, Any] JSON entry that holds additional information about the d1 maps """ @@ -114,4 +116,5 @@ class DepthOneMapTable(SQLModel, table=True): back_populates="maps", link_model=DepthOneToCoaddTable, ) + box: list[float] notes: dict[str, Any] | None = Field(default=None, sa_type=JSON) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index e4a6f2c..a2467f1 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,7 +2,7 @@ Sky coverage table. """ -from sqlmodel import Field, Relationship, Index, SQLModel +from sqlmodel import Field, Index, Relationship, SQLModel from .depth_one_map import DepthOneMapTable diff --git a/mapcat/database/tools.py b/mapcat/database/tools.py index 212bcab..8ec5035 100644 --- a/mapcat/database/tools.py +++ b/mapcat/database/tools.py @@ -1,10 +1,11 @@ import numpy as np -from pixell import enmap from numpy.typing import NDArray +from pixell import enmap from .depth_one_map import DepthOneMapTable from .sky_coverage import SkyCoverageTable + def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: """ Given the bounding box of map, return the list diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 806ca41..9eaae8b 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -84,6 +84,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT ctime=file_info["ctime"], start_time=file_info["start_time"], stop_time=file_info["stop_time"], + box=file_info["box"], tods=tods, ) From b451ed30c04b1b47ca2053ed63832142c3161bd1 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 13:38:41 -0500 Subject: [PATCH 05/35] fix: more ruff formating --- mapcat/database/sky_coverage.py | 29 ++++++++++++++-------------- mapcat/database/tools.py | 21 ++++++++++++++------ mapcat/toolkit/act.py | 19 +++++++++--------- tests/test_mapcat.py | 34 ++++++++++++++++----------------- 4 files changed, 57 insertions(+), 46 deletions(-) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index a2467f1..c885b94 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -9,16 +9,16 @@ class SkyCoverageTable(SQLModel, table=True): """ - Table for tracking sky coverage patches with non-zero overlap with a given depth one map. + Table for tracking sky coverage patches with non-zero overlap with a given depth one map. x and y are 0->36 and 0-18 respectively for CAR patches with 10x10 degrees each. Attributes ---------- id : uuid.UUID Unique ID of this coverage instance - map : DepthOneMapTable + map : DepthOneMapTable Depth 1 map being tracked. Foreign into DepthOneMap - map_id : int + map_id : int ID of depth 1 map being tracked x : int x-index of coverage patch. x=0 runs from RA 0 to 10,, etc. @@ -32,17 +32,18 @@ class SkyCoverageTable(SQLModel, table=True): y: int = Field(index=True, primary_key=True) map_id: int = Field( - foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE", primary_key=True, + foreign_key="depth_one_maps.map_id", + nullable=False, + ondelete="CASCADE", + primary_key=True, ) map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") - __table_args__ = ( - Index( - "ix_depth_one_sky_coverage_map_id_x_y", - "map_id", - "x", - "y", - unique=True, - pimary_key=True, - ) - ) \ No newline at end of file + __table_args__ = Index( + "ix_depth_one_sky_coverage_map_id_x_y", + "map_id", + "x", + "y", + unique=True, + pimary_key=True, + ) diff --git a/mapcat/database/tools.py b/mapcat/database/tools.py index 8ec5035..cfa603b 100644 --- a/mapcat/database/tools.py +++ b/mapcat/database/tools.py @@ -24,7 +24,7 @@ def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: A list of sky coverage tiles that cover the map """ - dec_min, ra_max = np.rad2deg(box[0]) + dec_min, ra_max = np.rad2deg(box[0]) dec_max, ra_min = np.rad2deg(box[1]) dec_min = np.floor(dec_min / 10) * 10 @@ -40,14 +40,20 @@ def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: for ra in ras: for dec in decs: - skybox = np.array([[np.deg2rad(dec), np.deg2rad(ra + 10)], [np.deg2rad(dec + 10), np.deg2rad(ra)]]) + skybox = np.array( + [ + [np.deg2rad(dec), np.deg2rad(ra + 10)], + [np.deg2rad(dec + 10), np.deg2rad(ra)], + ] + ) submap = enmap.submap(tmap, skybox) if np.any(submap): - ra_idx.append(int(ra/10)) - dec_id.append(int(dec/10) + 9) + ra_idx.append(int(ra / 10)) + dec_id.append(int(dec / 10) + 9) return list(zip(ra_idx, dec_id)) + def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: """ Get the list of sky coverage tiles that cover a given depth one map @@ -65,7 +71,10 @@ def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: tmap = enmap.read_map(d1map.mean_time_path) box = d1map.box - + coverage_tiles = get_sky_coverage(box, tmap) - return [SkyCoverageTable(x=tile[0], y=tile[1], map=d1map, map_id=d1map.map_id) for tile in coverage_tiles] \ No newline at end of file + return [ + SkyCoverageTable(x=tile[0], y=tile[1], map=d1map, map_id=d1map.map_id) + for tile in coverage_tiles + ] diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 9eaae8b..b57edf1 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -18,7 +18,7 @@ def extract_string(input: bytes) -> str: def parse_info_file(path: Path) -> dict: with h5py.File(path, "r") as f: - if 'band' in f and 'detset' in f: + if "band" in f and "detset" in f: ## backward compatibility with old SO format return { "frequency": extract_string(f["band"][...])[1:], @@ -32,14 +32,15 @@ def parse_info_file(path: Path) -> dict: else: ## ACT depth1 format. return { - "frequency": extract_string(f["array"][...]).split('_')[1], - "tube_slot": extract_string(f["array"][...]).split('_')[0], - "observations": [extract_string(x) for x in f["ids"][...]], - "start_time": f["period"][0], - "stop_time": f["period"][1], - "ctime": float(f["t"][...]), - "box": f["box"][...], - } + "frequency": extract_string(f["array"][...]).split("_")[1], + "tube_slot": extract_string(f["array"][...]).split("_")[0], + "observations": [extract_string(x) for x in f["ids"][...]], + "start_time": f["period"][0], + "stop_time": f["period"][1], + "ctime": float(f["t"][...]), + "box": f["box"][...], + } + def parse_filenames(base: str, relative_to: Path) -> dict[str, str]: """ diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index ba2c330..21dd08b 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -424,7 +424,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): with database_sessionmaker() as session: atomic = session.get(AtomicMapTable, atomic_map_id) atomic_coadds = atomic.coadds - + assert atomic.atomic_map_id == atomic_map_id assert atomic.obs_id == "obs_1755643930_satp3_1111111" assert atomic.telescope == "satp3" @@ -463,9 +463,9 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert atomic.wind_speed is None assert atomic.wind_direction is None assert atomic.rqu_avg is None - + assert atomic_coadds[0].coadd_id == daily_coadd_id - + # Create a weekly (parent) coadd with database_sessionmaker() as session: data = AtomicMapCoaddTable( @@ -478,7 +478,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", - child_coadds=[cmap] + child_coadds=[cmap], ) session.add(data) @@ -491,15 +491,15 @@ def test_create_atomic_map_coadd(database_sessionmaker): with database_sessionmaker() as session: cmap2 = session.get(AtomicMapCoaddTable, weekly_coadd_id) weekly_child_coadds = cmap2.child_coadds - + assert cmap2.coadd_id == weekly_coadd_id assert weekly_child_coadds[0].coadd_id == daily_coadd_id - + # Check daily coadd has weekly coadd as parent with database_sessionmaker() as session: cmap = session.get(AtomicMapCoaddTable, daily_coadd_id) daily_parent_coadds = cmap.parent_coadds - + assert daily_parent_coadds[0].coadd_id == weekly_coadd_id # Check bad map ID raises ValueError @@ -509,7 +509,7 @@ def test_create_atomic_map_coadd(database_sessionmaker): if result is None: raise ValueError("Map ID does not exist") - + def test_add_remove_atomic_map_coadd(database_sessionmaker): # Create daily and weekly coadds with database_sessionmaker() as session: @@ -535,7 +535,7 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", - child_coadds=[daily] + child_coadds=[daily], ) session.add_all( @@ -545,22 +545,22 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): ] ) session.commit() - + daily_coadd_id = daily.coadd_id weekly_coadd_id = weekly.coadd_id - + # Remove weekly coadd and check daily coadd parents with database_sessionmaker() as session: x = session.get(AtomicMapCoaddTable, weekly_coadd_id) session.delete(x) session.commit() - + with pytest.raises(ValueError): with database_sessionmaker() as session: daily = session.get(AtomicMapCoaddTable, daily_coadd_id) if len(daily.parent_coadds) == 0: raise ValueError("Daily map has no parent coadds") - + # Check in reverse with database_sessionmaker() as session: weekly2 = AtomicMapCoaddTable( @@ -573,21 +573,21 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): freq_channel="f090", geom_file_path="/PATH/TO/GEOM/FILE", split_label="full", - child_coadds=[daily] + child_coadds=[daily], ) session.add(weekly2) session.commit() weekly2_coadd_id = weekly2.coadd_id - + with database_sessionmaker() as session: x = session.get(AtomicMapCoaddTable, daily_coadd_id) session.delete(x) session.commit() - + with pytest.raises(ValueError): with database_sessionmaker() as session: weekly2 = session.get(AtomicMapCoaddTable, weekly2_coadd_id) if len(weekly2.child_coadds) == 0: - raise ValueError("Weekly map has no child coadds") \ No newline at end of file + raise ValueError("Weekly map has no child coadds") From e31e3f130f686943e8106b7548c5d860a63751f8 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 14:42:47 -0500 Subject: [PATCH 06/35] feat: switching from passing box to getting box using pixell --- mapcat/database/depth_one_map.py | 4 +--- mapcat/database/tools.py | 11 ++++------- mapcat/toolkit/act.py | 1 - 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index bc934e3..aa32033 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -66,8 +66,7 @@ class DepthOneMapTable(SQLModel, table=True): List of pipeline info associed with d1 map depth_one_sky_coverage : list[SkyCoverageTable] List of sky coverage patches for d1 map. - box : list[float] - Bounding box of map in format [[dec_min, ra_max], [dec_max, ra_min]], units in radians. + notes: dict[str, Any] JSON entry that holds additional information about the d1 maps """ @@ -116,5 +115,4 @@ class DepthOneMapTable(SQLModel, table=True): back_populates="maps", link_model=DepthOneToCoaddTable, ) - box: list[float] notes: dict[str, Any] | None = Field(default=None, sa_type=JSON) diff --git a/mapcat/database/tools.py b/mapcat/database/tools.py index cfa603b..a284af9 100644 --- a/mapcat/database/tools.py +++ b/mapcat/database/tools.py @@ -1,20 +1,17 @@ import numpy as np -from numpy.typing import NDArray from pixell import enmap from .depth_one_map import DepthOneMapTable from .sky_coverage import SkyCoverageTable -def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: +def get_sky_coverage(tmap: enmap.enmap) -> list: """ - Given the bounding box of map, return the list + Given the time map of a depth1 map, return the list of sky coverage tiles that cover that map Parameters ---------- - box : NDArray - The bounding box of the map, in the format [[dec_min, ra_max], [dec_max, ra_min]], units in radians tmap : enmap.enmap The time map of the depth-one map. Pixels that were observed have non-zero values. @@ -23,6 +20,7 @@ def get_sky_coverage(box: NDArray, tmap: enmap.enmap) -> list: tiles : list A list of sky coverage tiles that cover the map """ + box = tmap.box() dec_min, ra_max = np.rad2deg(box[0]) dec_max, ra_min = np.rad2deg(box[1]) @@ -70,9 +68,8 @@ def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: """ tmap = enmap.read_map(d1map.mean_time_path) - box = d1map.box - coverage_tiles = get_sky_coverage(box, tmap) + coverage_tiles = get_sky_coverage(tmap) return [ SkyCoverageTable(x=tile[0], y=tile[1], map=d1map, map_id=d1map.map_id) diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index b57edf1..0ba4448 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -85,7 +85,6 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT ctime=file_info["ctime"], start_time=file_info["start_time"], stop_time=file_info["stop_time"], - box=file_info["box"], tods=tods, ) From 90db8e436dce38e1e9ad0a4fd103b20ef52bd8a3 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 15:01:20 -0500 Subject: [PATCH 07/35] feat: moving sky coverage tools --- mapcat/{database/tools.py => toolkit/update_sky_coverage.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mapcat/{database/tools.py => toolkit/update_sky_coverage.py} (100%) diff --git a/mapcat/database/tools.py b/mapcat/toolkit/update_sky_coverage.py similarity index 100% rename from mapcat/database/tools.py rename to mapcat/toolkit/update_sky_coverage.py From 3d7ef768789ea359a98e33398ee22ff2c3614267 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 15:53:53 -0500 Subject: [PATCH 08/35] fix: fixing composite key generation --- mapcat/database/sky_coverage.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index c885b94..9ab307f 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -3,6 +3,7 @@ """ from sqlmodel import Field, Index, Relationship, SQLModel +from sqlalchemy import PrimaryKeyConstraint from .depth_one_map import DepthOneMapTable @@ -14,8 +15,8 @@ class SkyCoverageTable(SQLModel, table=True): Attributes ---------- - id : uuid.UUID - Unique ID of this coverage instance + sky_cov_id : PrimaryKeyConstraint + Composite ID from map_id, x, and y map : DepthOneMapTable Depth 1 map being tracked. Foreign into DepthOneMap map_id : int @@ -39,7 +40,13 @@ class SkyCoverageTable(SQLModel, table=True): ) map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") - __table_args__ = Index( + __table_args__ = ( + PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"), + ) + + """ + __table_args__ = ( + Index( "ix_depth_one_sky_coverage_map_id_x_y", "map_id", "x", @@ -47,3 +54,5 @@ class SkyCoverageTable(SQLModel, table=True): unique=True, pimary_key=True, ) + ) + """ From 9e30a6f9e31f3a31a311887b21934d6784297864 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 16:16:29 -0500 Subject: [PATCH 09/35] feat: starting skycoverage update function --- mapcat/database/sky_coverage.py | 6 ++---- mapcat/toolkit/update_sky_coverage.py | 27 +++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index 9ab307f..a002a2c 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -2,8 +2,8 @@ Sky coverage table. """ -from sqlmodel import Field, Index, Relationship, SQLModel from sqlalchemy import PrimaryKeyConstraint +from sqlmodel import Field, Relationship, SQLModel from .depth_one_map import DepthOneMapTable @@ -40,9 +40,7 @@ class SkyCoverageTable(SQLModel, table=True): ) map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") - __table_args__ = ( - PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"), - ) + __table_args__ = (PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"),) """ __table_args__ = ( diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index a284af9..9df9ded 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -1,8 +1,8 @@ import numpy as np from pixell import enmap -from .depth_one_map import DepthOneMapTable -from .sky_coverage import SkyCoverageTable +from ..database.depth_one_map import DepthOneMapTable +from ..database.sky_coverage import SkyCoverageTable def get_sky_coverage(tmap: enmap.enmap) -> list: @@ -75,3 +75,26 @@ def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: SkyCoverageTable(x=tile[0], y=tile[1], map=d1map, map_id=d1map.map_id) for tile in coverage_tiles ] + + +def main(): + from sqlalchemy import select + + from mapcat.helper import settings + + with settings.session() as session: + stmt = ( + select(DepthOneMapTable.map_name) + .select_from( + DepthOneMapTable.outerjoin( + SkyCoverageTable, + SkyCoverageTable.map_name == DepthOneMapTable.map_name, + ) + ) + .where(SkyCoverageTable.map_name.is_(None)) + ) + for d1map in session.execute(stmt): + SkyCov = coverage_from_depthone(d1map) + d1map.depth_one_sky_coverage = SkyCov + session.add(d1map) + session.commit() From 0064a1588a450ef9b158a3be8f3e73ab311dfedf Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 16:44:37 -0500 Subject: [PATCH 10/35] feat: add sqlite to gitignore --- .gitignore | 1 + mapcat/database/sky_coverage.py | 12 ------------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 1c4970c..54ee18a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .venv/* *.db +*.sqlite *__pycache__* dist/* *.egg-info diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index a002a2c..1ce0354 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -42,15 +42,3 @@ class SkyCoverageTable(SQLModel, table=True): __table_args__ = (PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"),) - """ - __table_args__ = ( - Index( - "ix_depth_one_sky_coverage_map_id_x_y", - "map_id", - "x", - "y", - unique=True, - pimary_key=True, - ) - ) - """ From dbe9283a675e95b1cd1f5bea402fdfa2eaeee9b7 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 12 Feb 2026 17:33:14 -0500 Subject: [PATCH 11/35] fix: updating outerjoin query --- mapcat/database/sky_coverage.py | 1 - mapcat/toolkit/update_sky_coverage.py | 23 ++++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index 1ce0354..d9b76bb 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -41,4 +41,3 @@ class SkyCoverageTable(SQLModel, table=True): map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") __table_args__ = (PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"),) - diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index 9df9ded..49b4fb3 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -1,8 +1,8 @@ import numpy as np from pixell import enmap -from ..database.depth_one_map import DepthOneMapTable -from ..database.sky_coverage import SkyCoverageTable +from mapcat.database.depth_one_map import DepthOneMapTable +from mapcat.database.sky_coverage import SkyCoverageTable def get_sky_coverage(tmap: enmap.enmap) -> list: @@ -78,22 +78,19 @@ def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: def main(): - from sqlalchemy import select - from mapcat.helper import settings with settings.session() as session: - stmt = ( - select(DepthOneMapTable.map_name) - .select_from( - DepthOneMapTable.outerjoin( - SkyCoverageTable, - SkyCoverageTable.map_name == DepthOneMapTable.map_name, - ) + d1maps = ( + session.query(DepthOneMapTable) + .outerjoin( + SkyCoverageTable, SkyCoverageTable.map_id == DepthOneMapTable.map_id ) - .where(SkyCoverageTable.map_name.is_(None)) + .filter(SkyCoverageTable.map_id.is_(None)) + .all() ) - for d1map in session.execute(stmt): + for d1map in d1maps: + print(d1map) SkyCov = coverage_from_depthone(d1map) d1map.depth_one_sky_coverage = SkyCov session.add(d1map) From 98d5d85eaac0ee2864b362c881d816112e1945e3 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 13 Feb 2026 17:23:21 -0500 Subject: [PATCH 12/35] feat: move core functionaliy of ingestact to a separate function for testability --- mapcat/toolkit/act.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 0ba4448..6add4cf 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -2,15 +2,14 @@ Create objects from an ACT depth 1 map file. """ +import argparse as ap from pathlib import Path import h5py +from sqlalchemy.orm import sessionmaker from mapcat.database import DepthOneMapTable, TODDepthOneTable -# from astropy import units as u -# from astropy.coordinates import SkyCoord - def extract_string(input: bytes) -> str: return str(input).replace("b'", "").replace("'", "") @@ -130,9 +129,28 @@ def glob(input_glob: str, relative_to: Path, telescope: str) -> list[DepthOneMap """ -def main(): - import argparse as ap +def core(session: sessionmaker, args: ap.Namespace): + """ + Driver function for act.py Takes a session and a arg parser + and creates DepthOneMapTable objects from the files listed + matching the glob patter in the parser and adds them to the + database in session. + + Parameters + ---------- + session : sessionmaker + A SQLAlchemy sessionmaker to use for database access. + args : argparse.Namespace + Parsed args with the glob patterns to match. + """ + with session() as session: + maps = glob(args.glob, args.relative_to, args.telescope) + session.add_all(maps) + session.commit() + + +def main(): from mapcat.helper import settings parser = ap.ArgumentParser(prog="actingest", usage=USAGE, description=HELP_TEXT) @@ -163,7 +181,4 @@ def main(): args = parser.parse_args() - with settings.session() as session: - maps = glob(args.glob, args.relative_to, args.telescope) - session.add_all(maps) - session.commit() + core(session=settings.session, args=args) From 5649579dfc40ae6b5325ca4828dbcc1b3d71689c Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 13 Feb 2026 17:24:05 -0500 Subject: [PATCH 13/35] feat: move core functionality of update_sky_coverage to its own function for testability --- mapcat/toolkit/update_sky_coverage.py | 50 ++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index 49b4fb3..5138b62 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -1,8 +1,29 @@ +from pathlib import Path + import numpy as np from pixell import enmap from mapcat.database.depth_one_map import DepthOneMapTable from mapcat.database.sky_coverage import SkyCoverageTable +from mapcat.helper import settings + + +def resolve_tmap(d1table: DepthOneMapTable) -> Path: + """ + Resolve the local path to a tmap from a d1 table. + + Parameters + ---------- + d1table : DepthOneMapTable + The depth one map table to resolve the tmap for + + Returns + ------- + tmap_path : Path + The local path to the tmap for the depth one map + """ + tmap_path = settings.depth_one_parent / d1table.mean_time_path + return tmap_path def get_sky_coverage(tmap: enmap.enmap) -> list: @@ -52,7 +73,7 @@ def get_sky_coverage(tmap: enmap.enmap) -> list: return list(zip(ra_idx, dec_id)) -def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: +def coverage_from_depthone(d1table: DepthOneMapTable) -> list[SkyCoverageTable]: """ Get the list of sky coverage tiles that cover a given depth one map @@ -66,21 +87,27 @@ def coverage_from_depthone(d1map: DepthOneMapTable) -> list[SkyCoverageTable]: tiles : list[SkyCoverageTable] A list of sky coverage tiles that cover the map """ - - tmap = enmap.read_map(d1map.mean_time_path) + tmap_path = resolve_tmap(d1table) + tmap = enmap.read_map(str(tmap_path)) coverage_tiles = get_sky_coverage(tmap) return [ - SkyCoverageTable(x=tile[0], y=tile[1], map=d1map, map_id=d1map.map_id) + SkyCoverageTable(x=tile[0], y=tile[1], map=d1table, map_id=d1table.map_id) for tile in coverage_tiles ] -def main(): - from mapcat.helper import settings +def core(session): + """ + Core function for updating the sky coverage table. For each depth one map that does not have any associated sky coverage tiles, compute the sky coverage tiles and add them to the database. - with settings.session() as session: + Parameters + ---------- + session : sessionmaker + A SQLAlchemy sessionmaker to use for database access. + """ + with session() as session: d1maps = ( session.query(DepthOneMapTable) .outerjoin( @@ -90,8 +117,13 @@ def main(): .all() ) for d1map in d1maps: - print(d1map) SkyCov = coverage_from_depthone(d1map) + for cov in SkyCov: + session.add(cov) + session.commit() d1map.depth_one_sky_coverage = SkyCov - session.add(d1map) session.commit() + + +def main(): + core(session=settings.session) From 281f19c3490576dfcfbd133c9269e2e7a6a36f1a Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 13 Feb 2026 17:24:30 -0500 Subject: [PATCH 14/35] feat: adding test for ingestact and update_sky_coverage --- tests/test_act.py | 234 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 tests/test_act.py diff --git a/tests/test_act.py b/tests/test_act.py new file mode 100644 index 0000000..12b26a3 --- /dev/null +++ b/tests/test_act.py @@ -0,0 +1,234 @@ +import argparse as ap +import os +from pathlib import Path + +import pytest +import requests +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +import mapcat.toolkit.act as act +import mapcat.toolkit.update_sky_coverage as update_sky_coverage +from mapcat.database import DepthOneMapTable + +DATA_URLS = [ + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_info.hdf", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_ivar.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_kappa.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_map.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_rho.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_time.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505646390_pa6_f150_info.hdf", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505646390_pa6_f150_ivar.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505646390_pa6_f150_kappa.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505646390_pa6_f150_map.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505646390_pa6_f150_rho.fits", + "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505646390_pa6_f150_time.fits", +] + +cov_mapping = { + "1505603190": [ + (-4, 3), + (-4, 4), + (-4, 6), + (-4, 7), + (-3, 3), + (-3, 4), + (-3, 5), + (-3, 6), + (-3, 7), + (-2, 3), + (-2, 4), + (-2, 5), + (-2, 6), + (-2, 7), + (-1, 3), + (-1, 4), + (-1, 5), + (-1, 6), + (-1, 7), + (0, 3), + (0, 4), + (0, 5), + (0, 6), + (0, 7), + (1, 3), + (1, 4), + (1, 5), + (1, 6), + (1, 7), + (2, 3), + (2, 4), + (2, 5), + (2, 6), + (2, 7), + (3, 3), + (3, 4), + (3, 5), + (3, 6), + (3, 7), + (4, 3), + (4, 4), + (4, 5), + (4, 6), + (4, 7), + (5, 3), + (5, 4), + (5, 5), + (5, 6), + (5, 7), + (6, 3), + (6, 4), + (6, 5), + (6, 6), + (6, 7), + (7, 3), + (7, 4), + (7, 5), + (7, 6), + (7, 7), + (8, 3), + (8, 4), + (8, 5), + (8, 6), + (8, 7), + (9, 3), + (9, 4), + (9, 5), + (9, 6), + (9, 7), + (10, 3), + (10, 4), + (10, 5), + (10, 6), + (10, 7), + (11, 3), + (11, 4), + (11, 5), + (11, 6), + ], + "1505646390": [(4, 3), (4, 4), (4, 5), (5, 3), (5, 4), (5, 5)], +} + + +def run_migration(database_path: str): + """ + Run the migration on the database. + """ + from alembic import command + from alembic.config import Config + + alembic_cfg = Config( + __file__.replace("tests/test_act.py", "") + "mapcat/alembic.ini" + ) + database_url = f"sqlite:///{database_path}" + alembic_cfg.set_main_option("sqlalchemy.url", database_url) + command.upgrade(alembic_cfg, "head") + + return + + +@pytest.fixture(scope="session", autouse=True) +def database_sessionmaker(tmp_path_factory): + """ + Create a temporary SQLite database for testing. + """ + + tmp_path = tmp_path_factory.mktemp("mapcat") + # Create a temporary SQLite database for testing. + database_path = tmp_path / "test.db" + + # Run the migration on the database. This is blocking. + run_migration(database_path) + + database_url = f"sqlite:///{database_path}" + + engine = create_engine(database_url, echo=True, future=True) + + yield sessionmaker(bind=engine, expire_on_commit=False) + + # Clean up the database (don't do this in case we want to inspect) + database_path.unlink() + + +@pytest.fixture(scope="session") +def downloaded_data_file(request): + """ + Fixture to download depth 1 maps for testing. + + Parameters + ---------- + tmp_path_factory : pytest.TempPathFactory + Factory to create temporary directory for downloaded files + Returns + ------- + file_path : str + Path to the downloaded file + """ + + for url in DATA_URLS: + cache_dir = Path("../.pytest_cache/d1maps") + filename = os.path.basename(url) + subdir = os.path.basename(os.path.dirname(url)) + file_path = cache_dir / subdir / filename + + # Check to see if each file is already downloaded in the cache, if so skip downloading it again + cached_file = request.config.cache.get( + "downloaded_file_" + os.path.basename(url), None + ) + if cached_file and os.path.exists(cached_file): + continue + + # Download the file + response = requests.get(url) + response.raise_for_status() + + # To match the expected directory strcutre, e.g. 15060/ contains depth 1 maps + # starting at 15060, make the intermediate directory + if not file_path.parent.exists(): + file_path.parent.mkdir(parents=True, exist_ok=True) + + with open(file_path, "wb") as f: + f.write(response.content) + + # Set the location of the file in the cache + request.config.cache.set( + "downloaded_file_" + os.path.basename(url), str(file_path) + ) + + return cache_dir + + +def test_act(database_sessionmaker, downloaded_data_file): + args = ap.Namespace( + glob="*/*_map.fits", + relative_to=downloaded_data_file, + telescope="act", + ) + + act.core(session=database_sessionmaker, args=args) + + with database_sessionmaker() as session: + maps = session.query(DepthOneMapTable).all() + assert len(maps) == 2 + + # Not the prettiest tests of all time, but whatever. + for map in maps: + assert map.tube_slot in ["pa4", "pa6"] + assert map.frequency == "f150" + assert map.ctime in [1505603190, 1505646390] + + +def test_sky_coverage(database_sessionmaker): + with database_sessionmaker() as session: + update_sky_coverage.core(session=database_sessionmaker) + + d1maps = session.query(DepthOneMapTable).all() + print("LEN", len(d1maps)) + for d1map in d1maps: + print("\n\n\n\nD1MAP: ", d1map.depth_one_sky_coverage) + assert len(d1map.depth_one_sky_coverage) > 0 + for cov in d1map.depth_one_sky_coverage: + # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. + # The coverage tiles are stored in cov_mapping, which is a dict mapping from ctime to a list of (x,y) tuples representing the coverage tiles. + assert tuple(cov.x, cov.y) in cov_mapping[str(d1map.ctime)] From 0d445ca53dc5c54facac37506ff26b4684f1b91d Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Mon, 16 Feb 2026 11:29:08 -0500 Subject: [PATCH 15/35] fix: map=d1table is redundant --- mapcat/toolkit/update_sky_coverage.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index 5138b62..ee7ac0b 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -7,6 +7,8 @@ from mapcat.database.sky_coverage import SkyCoverageTable from mapcat.helper import settings +from sqlalchemy import update + def resolve_tmap(d1table: DepthOneMapTable) -> Path: """ @@ -93,7 +95,7 @@ def coverage_from_depthone(d1table: DepthOneMapTable) -> list[SkyCoverageTable]: coverage_tiles = get_sky_coverage(tmap) return [ - SkyCoverageTable(x=tile[0], y=tile[1], map=d1table, map_id=d1table.map_id) + SkyCoverageTable(x=tile[0], y=tile[1], map_id=d1table.map_id) for tile in coverage_tiles ] @@ -118,11 +120,12 @@ def core(session): ) for d1map in d1maps: SkyCov = coverage_from_depthone(d1map) - for cov in SkyCov: - session.add(cov) - session.commit() - d1map.depth_one_sky_coverage = SkyCov - session.commit() + session.add_all(SkyCov) + update_stmt = (update(DepthOneMapTable) + .where(DepthOneMapTable.map_id == d1map.map_id) + .values(depth_one_sky_coverage=SkyCov)) + session.execute(update_stmt) + session.commit() def main(): From a18fb0bd624260334ba21e48ac2d817f3e831e14 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Mon, 16 Feb 2026 11:29:35 -0500 Subject: [PATCH 16/35] fix: move update_sky_coverage --- tests/test_act.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_act.py b/tests/test_act.py index 12b26a3..ffded99 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -220,8 +220,9 @@ def test_act(database_sessionmaker, downloaded_data_file): def test_sky_coverage(database_sessionmaker): + update_sky_coverage.core(session=database_sessionmaker) with database_sessionmaker() as session: - update_sky_coverage.core(session=database_sessionmaker) + d1maps = session.query(DepthOneMapTable).all() print("LEN", len(d1maps)) From bbe3fe6ba1cef29afc6d68697a0c0ac86b59de81 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Mon, 16 Feb 2026 11:30:01 -0500 Subject: [PATCH 17/35] fix: ruff --- mapcat/toolkit/update_sky_coverage.py | 11 ++++++----- tests/test_act.py | 2 -- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index ee7ac0b..9c0f7c0 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -2,13 +2,12 @@ import numpy as np from pixell import enmap +from sqlalchemy import update from mapcat.database.depth_one_map import DepthOneMapTable from mapcat.database.sky_coverage import SkyCoverageTable from mapcat.helper import settings -from sqlalchemy import update - def resolve_tmap(d1table: DepthOneMapTable) -> Path: """ @@ -121,9 +120,11 @@ def core(session): for d1map in d1maps: SkyCov = coverage_from_depthone(d1map) session.add_all(SkyCov) - update_stmt = (update(DepthOneMapTable) - .where(DepthOneMapTable.map_id == d1map.map_id) - .values(depth_one_sky_coverage=SkyCov)) + update_stmt = ( + update(DepthOneMapTable) + .where(DepthOneMapTable.map_id == d1map.map_id) + .values(depth_one_sky_coverage=SkyCov) + ) session.execute(update_stmt) session.commit() diff --git a/tests/test_act.py b/tests/test_act.py index ffded99..fd8ce71 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -222,8 +222,6 @@ def test_act(database_sessionmaker, downloaded_data_file): def test_sky_coverage(database_sessionmaker): update_sky_coverage.core(session=database_sessionmaker) with database_sessionmaker() as session: - - d1maps = session.query(DepthOneMapTable).all() print("LEN", len(d1maps)) for d1map in d1maps: From c9c046f95194f4dadc087212254c1d06c44c0bed Mon Sep 17 00:00:00 2001 From: Josh Borrow Date: Mon, 16 Feb 2026 17:00:49 -0500 Subject: [PATCH 18/35] Fix tests --- .../78293df04081_create_depth_1_map_table.py | 6 +++--- mapcat/database/sky_coverage.py | 1 + pyproject.toml | 3 ++- tests/test_act.py | 14 ++++++++++---- tests/test_mapcat.py | 15 +++++++-------- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py b/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py index 54c31af..cb5ae6e 100644 --- a/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py +++ b/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py @@ -139,14 +139,14 @@ def upgrade() -> None: op.create_table( "depth_one_sky_coverage", - sa.Column("patch_id", sa.Integer, primary_key=True), - sa.Column("x", sa.CHAR, nullable=False), - sa.Column("y", sa.CHAR, nullable=False), + sa.Column("x", sa.CHAR, nullable=False, primary_key=True), + sa.Column("y", sa.CHAR, nullable=False, primary_key=True), sa.Column( "map_id", sa.Integer, sa.ForeignKey("depth_one_maps.map_id", ondelete="CASCADE"), nullable=False, + primary_key=True, ), ) diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py index d9b76bb..fa5021f 100644 --- a/mapcat/database/sky_coverage.py +++ b/mapcat/database/sky_coverage.py @@ -38,6 +38,7 @@ class SkyCoverageTable(SQLModel, table=True): ondelete="CASCADE", primary_key=True, ) + map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage") __table_args__ = (PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"),) diff --git a/pyproject.toml b/pyproject.toml index e665473..583af5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dependencies = [ "sqlalchemy[asyncio]", "aiosqlite", "alembic", - "pydantic_settings" + "pydantic_settings", + "pixell", ] [project.scripts] diff --git a/tests/test_act.py b/tests/test_act.py index fd8ce71..9528b60 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -136,7 +136,7 @@ def database_sessionmaker(tmp_path_factory): tmp_path = tmp_path_factory.mktemp("mapcat") # Create a temporary SQLite database for testing. - database_path = tmp_path / "test.db" + database_path = tmp_path / "act_test.db" # Run the migration on the database. This is blocking. run_migration(database_path) @@ -218,16 +218,22 @@ def test_act(database_sessionmaker, downloaded_data_file): assert map.frequency == "f150" assert map.ctime in [1505603190, 1505646390] + # Clean up, otherewise we interfere with test_sky_coverage + session.delete(map) + + session.commit() + def test_sky_coverage(database_sessionmaker): update_sky_coverage.core(session=database_sessionmaker) with database_sessionmaker() as session: d1maps = session.query(DepthOneMapTable).all() - print("LEN", len(d1maps)) for d1map in d1maps: - print("\n\n\n\nD1MAP: ", d1map.depth_one_sky_coverage) assert len(d1map.depth_one_sky_coverage) > 0 for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. # The coverage tiles are stored in cov_mapping, which is a dict mapping from ctime to a list of (x,y) tuples representing the coverage tiles. - assert tuple(cov.x, cov.y) in cov_mapping[str(d1map.ctime)] + assert ( + cov.x, + cov.y, + ) in cov_mapping[str(d1map.ctime)] diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index 21dd08b..b95f198 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -35,7 +35,7 @@ def run_migration(database_path: str): return -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="module", autouse=True) def database_sessionmaker(tmp_path_factory): """ Create a temporary SQLite database for testing. @@ -98,11 +98,11 @@ def test_create_depth_one(database_sessionmaker): processing_start=1756787524.0, processing_end=1756797524.0, processing_status="done", - map=dmap, + map_id=map_id, ) pointing_residual = PointingResidualTable( - ra_offset=1.2, dec_offset=-0.8, map=dmap + ra_offset=1.2, dec_offset=-0.8, map_id=map_id ) tod = TODDepthOneTable( @@ -136,11 +136,11 @@ def test_create_depth_one(database_sessionmaker): sotodlib_version="1.2.3", map_maker="minkasi", preprocess_info={"config": "test"}, - map=dmap, + map_id=map_id, ) sky_coverage = SkyCoverageTable( - map=dmap, + map_id=map_id, x=5, y=2, ) @@ -154,7 +154,7 @@ def test_create_depth_one(database_sessionmaker): point_id = pointing_residual.pointing_residual_id tod_id = tod.tod_id pipe_id = pipeline_info.pipeline_information_id - sky_id = sky_coverage.patch_id + sky_id = {"x": 5, "y": 2, "map_id": map_id} # Get child tables back with database_sessionmaker() as session: @@ -206,7 +206,6 @@ def test_create_depth_one(database_sessionmaker): assert pipe.map_maker == "minkasi" assert pipe.preprocess_info == {"config": "test"} - assert sky.patch_id == sky_id assert sky.x == "5" assert sky.y == "2" @@ -299,7 +298,7 @@ def test_add_remove_child_tables(database_sessionmaker): point_id = pointing_residual.pointing_residual_id tod_id = tod.tod_id pipe_id = pipeline_info.pipeline_information_id - sky_id = sky_coverage.patch_id + sky_id = {"x": 5, "y": 2, "map_id": dmap_id} # Check the cascades work with database_sessionmaker() as session: From 718265b94f1e0ab911b374b667fcacd242336f4d Mon Sep 17 00:00:00 2001 From: Josh Borrow Date: Mon, 16 Feb 2026 17:01:39 -0500 Subject: [PATCH 19/35] Add h5py as dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 583af5b..f701ceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "alembic", "pydantic_settings", "pixell", + "h5py" ] [project.scripts] From c33501e46baaa744e8b680e0c1c33d1886a724d2 Mon Sep 17 00:00:00 2001 From: Josh Borrow Date: Mon, 16 Feb 2026 17:04:20 -0500 Subject: [PATCH 20/35] Satisfy tyg --- mapcat/helper.py | 8 ++++---- mapcat/toolkit/update_sky_coverage.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mapcat/helper.py b/mapcat/helper.py index c8eee87..58dcb1c 100644 --- a/mapcat/helper.py +++ b/mapcat/helper.py @@ -16,10 +16,10 @@ class Settings(BaseSettings): # pragma: no cover database_name: str = "mapcat.db" database_type: Literal["sqlite", "postgresql"] = "sqlite" - depth_one_coadd_parent: Path = "./" - depth_one_parent: Path = "./" - atomic_coadd_parent: Path = "./" - atomic_parent: Path = "./" + depth_one_coadd_parent: Path = Path("./") + depth_one_parent: Path = Path("./") + atomic_coadd_parent: Path = Path("./") + atomic_parent: Path = Path("./") echo: bool = False diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index 9c0f7c0..e530c51 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -27,7 +27,7 @@ def resolve_tmap(d1table: DepthOneMapTable) -> Path: return tmap_path -def get_sky_coverage(tmap: enmap.enmap) -> list: +def get_sky_coverage(tmap: enmap.ndmap) -> list: """ Given the time map of a depth1 map, return the list of sky coverage tiles that cover that map From 0fccbb6daa6fa9dad1b7d0f0be2e51ecb61722aa Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Tue, 17 Feb 2026 16:51:41 -0500 Subject: [PATCH 21/35] fix: remove unneccesary update statement --- mapcat/toolkit/update_sky_coverage.py | 7 +------ tests/test_act.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index e530c51..2aa753f 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -120,12 +120,7 @@ def core(session): for d1map in d1maps: SkyCov = coverage_from_depthone(d1map) session.add_all(SkyCov) - update_stmt = ( - update(DepthOneMapTable) - .where(DepthOneMapTable.map_id == d1map.map_id) - .values(depth_one_sky_coverage=SkyCov) - ) - session.execute(update_stmt) + session.commit() diff --git a/tests/test_act.py b/tests/test_act.py index 9528b60..cbad8ee 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -27,7 +27,7 @@ ] cov_mapping = { - "1505603190": [ + "1505603190.0": [ (-4, 3), (-4, 4), (-4, 6), @@ -107,7 +107,7 @@ (11, 5), (11, 6), ], - "1505646390": [(4, 3), (4, 4), (4, 5), (5, 3), (5, 4), (5, 5)], + "1505646390.0": [(4, 3), (4, 4), (4, 5), (5, 3), (5, 4), (5, 5)], } @@ -158,7 +158,7 @@ def downloaded_data_file(request): Parameters ---------- - tmp_path_factory : pytest.TempPathFactory + request : pytest.FixtureRequest Factory to create temporary directory for downloaded files Returns ------- @@ -224,16 +224,27 @@ def test_act(database_sessionmaker, downloaded_data_file): session.commit() -def test_sky_coverage(database_sessionmaker): +def test_sky_coverage(database_sessionmaker, downloaded_data_file): + args = ap.Namespace( + glob="*/*_map.fits", + relative_to=downloaded_data_file, + telescope="act", + ) + act.core(session=database_sessionmaker, args=args) + update_sky_coverage.core(session=database_sessionmaker) with database_sessionmaker() as session: d1maps = session.query(DepthOneMapTable).all() for d1map in d1maps: + print("\n\n D1MAP: ", d1map) + print("\n\n") assert len(d1map.depth_one_sky_coverage) > 0 for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. # The coverage tiles are stored in cov_mapping, which is a dict mapping from ctime to a list of (x,y) tuples representing the coverage tiles. assert ( - cov.x, - cov.y, + int( + cov.x + ), # These should be ints, idk why I have to cast them (from str) + int(cov.y), ) in cov_mapping[str(d1map.ctime)] From 3311448b971fe25e1e6473ac33c7aa7974a3bc5b Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Tue, 17 Feb 2026 17:15:07 -0500 Subject: [PATCH 22/35] fix: remove unused imports --- mapcat/toolkit/update_sky_coverage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index 2aa753f..bfe7470 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -2,7 +2,6 @@ import numpy as np from pixell import enmap -from sqlalchemy import update from mapcat.database.depth_one_map import DepthOneMapTable from mapcat.database.sky_coverage import SkyCoverageTable @@ -120,7 +119,7 @@ def core(session): for d1map in d1maps: SkyCov = coverage_from_depthone(d1map) session.add_all(SkyCov) - + session.commit() From a87789668e269a887ff2423eaa7fc4dae921a4f7 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 20 Feb 2026 14:03:39 -0500 Subject: [PATCH 23/35] feat: switch to env variables for file paths --- .github/workflows/pytest.yml | 5 ++++- mapcat/helper.py | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 19ab424..f0f536d 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -2,6 +2,9 @@ name: Run tests on: [push] +env: + DEPTH_ONE_PATH: "../.pytest_cache/d1maps/" + jobs: runtest: env: @@ -29,4 +32,4 @@ jobs: - name: Run tests run: pytest --cov - \ No newline at end of file + diff --git a/mapcat/helper.py b/mapcat/helper.py index 58dcb1c..a16838d 100644 --- a/mapcat/helper.py +++ b/mapcat/helper.py @@ -2,6 +2,7 @@ Helper for mapcat database. """ +import os from pathlib import Path from typing import Literal @@ -16,10 +17,10 @@ class Settings(BaseSettings): # pragma: no cover database_name: str = "mapcat.db" database_type: Literal["sqlite", "postgresql"] = "sqlite" - depth_one_coadd_parent: Path = Path("./") - depth_one_parent: Path = Path("./") - atomic_coadd_parent: Path = Path("./") - atomic_parent: Path = Path("./") + depth_one_coadd_parent: Path = Path(os.environ.get("DEPTH_ONE_COADD_PARENT", "./")) + depth_one_parent: Path = Path(os.environ.get("DEPTH_ONE_PARENT", "./")) + atomic_coadd_parent: Path = Path(os.environ.get("ATOMIC_COADD_PARENT", "./")) + atomic_parent: Path = Path(os.environ.get("ATOMIC_PARENT", "./")) echo: bool = False From 4d5366a3a349997d60167914405084ebbf31f01e Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 20 Feb 2026 14:14:45 -0500 Subject: [PATCH 24/35] fix: updating to latest ruff --- ..._add_notes_missing_field_to_atomic_maps.py | 8 +- mapcat/alembic/versions/57b97937e1bc_.py | 8 +- .../6ce7e94dfd2d_add_filtered_masps.py | 12 +-- .../78293df04081_create_depth_1_map_table.py | 8 +- .../versions/976527c198bd_add_indices.py | 8 +- ..._add_notes_json_field_to_depth_one_maps.py | 8 +- ...670a1fdbe_update_atomic_map_coadd_table.py | 8 +- mapcat/database/__init__.py | 8 +- mapcat/database/pipeline_information.py | 2 - mapcat/helper.py | 2 - mapcat/toolkit/act.py | 10 +-- mapcat/toolkit/mapmaking.py | 5 +- mapcat/toolkit/update_sky_coverage.py | 13 ++-- tests/test_act.py | 5 +- tests/test_mapcat.py | 74 ++++++++----------- tests/test_mapmaking.py | 2 - 16 files changed, 77 insertions(+), 104 deletions(-) diff --git a/mapcat/alembic/versions/1195d17201ba_add_notes_missing_field_to_atomic_maps.py b/mapcat/alembic/versions/1195d17201ba_add_notes_missing_field_to_atomic_maps.py index ce59d4b..6f598fd 100644 --- a/mapcat/alembic/versions/1195d17201ba_add_notes_missing_field_to_atomic_maps.py +++ b/mapcat/alembic/versions/1195d17201ba_add_notes_missing_field_to_atomic_maps.py @@ -6,16 +6,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "1195d17201ba" -down_revision: Union[str, None] = "bc2be980cfe7" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = "bc2be980cfe7" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/mapcat/alembic/versions/57b97937e1bc_.py b/mapcat/alembic/versions/57b97937e1bc_.py index 4de2cf0..10c394b 100644 --- a/mapcat/alembic/versions/57b97937e1bc_.py +++ b/mapcat/alembic/versions/57b97937e1bc_.py @@ -6,16 +6,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "57b97937e1bc" -down_revision: Union[str, None] = "78293df04081" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = "78293df04081" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/mapcat/alembic/versions/6ce7e94dfd2d_add_filtered_masps.py b/mapcat/alembic/versions/6ce7e94dfd2d_add_filtered_masps.py index 3d15642..812c930 100644 --- a/mapcat/alembic/versions/6ce7e94dfd2d_add_filtered_masps.py +++ b/mapcat/alembic/versions/6ce7e94dfd2d_add_filtered_masps.py @@ -6,16 +6,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "6ce7e94dfd2d" -down_revision: Union[str, None] = "976527c198bd" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = "976527c198bd" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None NEW_COLUMNS_DEPTH_ONE = [ "rho_path", @@ -51,8 +51,6 @@ def upgrade() -> None: "depth_one_coadds", sa.Column(column_name, sa.String(), nullable=True) ) - pass - def downgrade() -> None: # Depth 1 @@ -63,5 +61,3 @@ def downgrade() -> None: for column_name in NEW_COLUMNS_DEPTH_ONE_COADDS: op.drop_column("depth_one_coadds", column_name) - - pass diff --git a/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py b/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py index cb5ae6e..318b770 100644 --- a/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py +++ b/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py @@ -7,16 +7,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "78293df04081" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/mapcat/alembic/versions/976527c198bd_add_indices.py b/mapcat/alembic/versions/976527c198bd_add_indices.py index ae49bdd..18ba635 100644 --- a/mapcat/alembic/versions/976527c198bd_add_indices.py +++ b/mapcat/alembic/versions/976527c198bd_add_indices.py @@ -6,15 +6,15 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence from alembic import op # revision identifiers, used by Alembic. revision: str = "976527c198bd" -down_revision: Union[str, None] = "fd6670a1fdbe" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = "fd6670a1fdbe" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/mapcat/alembic/versions/bc2be980cfe7_add_notes_json_field_to_depth_one_maps.py b/mapcat/alembic/versions/bc2be980cfe7_add_notes_json_field_to_depth_one_maps.py index 9834d06..202e25b 100644 --- a/mapcat/alembic/versions/bc2be980cfe7_add_notes_json_field_to_depth_one_maps.py +++ b/mapcat/alembic/versions/bc2be980cfe7_add_notes_json_field_to_depth_one_maps.py @@ -6,16 +6,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "bc2be980cfe7" -down_revision: Union[str, None] = "57b97937e1bc" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = "57b97937e1bc" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py b/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py index c185e1a..87438ca 100644 --- a/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py +++ b/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py @@ -6,16 +6,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "fd6670a1fdbe" -down_revision: Union[str, None] = "1195d17201ba" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = "1195d17201ba" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py index aa58b96..94c742a 100644 --- a/mapcat/database/__init__.py +++ b/mapcat/database/__init__.py @@ -13,14 +13,14 @@ from .tod import TODDepthOneTable __all__ = [ - "AtomicMapTable", "AtomicMapCoaddTable", + "AtomicMapTable", "DepthOneMapTable", - "TimeDomainProcessingTable", - "PointingResidualTable", - "TODDepthOneTable", "PipelineInformationTable", + "PointingResidualTable", "SkyCoverageTable", + "TODDepthOneTable", + "TimeDomainProcessingTable", ] ALL_TABLES = [ diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py index 79809a5..3f47ee0 100644 --- a/mapcat/database/pipeline_information.py +++ b/mapcat/database/pipeline_information.py @@ -36,5 +36,3 @@ class PipelineInformationTable(SQLModel, table=True): sotodlib_version: str map_maker: str preprocess_info: dict[str, Any] = Field(sa_type=JSON) - - map: DepthOneMapTable = Relationship(back_populates="pipeline_information") diff --git a/mapcat/helper.py b/mapcat/helper.py index a16838d..6efc7d6 100644 --- a/mapcat/helper.py +++ b/mapcat/helper.py @@ -75,5 +75,3 @@ def migrate(): location = __file__.replace("helper.py", "alembic.ini") subprocess.call(["alembic", "-c", location, "upgrade", "head"]) - - return diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py index 6add4cf..5370fa2 100644 --- a/mapcat/toolkit/act.py +++ b/mapcat/toolkit/act.py @@ -74,7 +74,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT for obs_id in file_info["observations"] ] - depth_one_map = DepthOneMapTable( + return DepthOneMapTable( map_name=filenames["map"].replace("_map.fits", ""), map_path=filenames["map"], ivar_path=filenames.get("ivar"), @@ -87,8 +87,6 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT tods=tods, ) - return depth_one_map - def glob(input_glob: str, relative_to: Path, telescope: str) -> list[DepthOneMapTable]: maps = [] @@ -144,10 +142,10 @@ def core(session: sessionmaker, args: ap.Namespace): Parsed args with the glob patterns to match. """ - with session() as session: + with session() as cur_session: maps = glob(args.glob, args.relative_to, args.telescope) - session.add_all(maps) - session.commit() + cur_session.add_all(maps) + cur_session.commit() def main(): diff --git a/mapcat/toolkit/mapmaking.py b/mapcat/toolkit/mapmaking.py index 6b70e47..4c98f24 100644 --- a/mapcat/toolkit/mapmaking.py +++ b/mapcat/toolkit/mapmaking.py @@ -34,8 +34,7 @@ def maps_containing_obs(obs_id: str, session: Session) -> list[DepthOneMapTable] if len(tod) == 0: # pragma: no cover raise ValueError(f"No TODs with obs ID {obs_id} found.") - depth_one_maps = tod[0].maps - return depth_one_maps + return tod[0].maps def build_obslists( @@ -67,4 +66,4 @@ def build_obslists( else: map_dict[obs_id] = map_list - return tuple((map_dict, no_map_list)) + return (map_dict, no_map_list) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index bfe7470..ec77a19 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -19,11 +19,10 @@ def resolve_tmap(d1table: DepthOneMapTable) -> Path: Returns ------- - tmap_path : Path + ettings.depth_one_parent / d1table.mean_time_path : Path The local path to the tmap for the depth one map """ - tmap_path = settings.depth_one_parent / d1table.mean_time_path - return tmap_path + return settings.depth_one_parent / d1table.mean_time_path def get_sky_coverage(tmap: enmap.ndmap) -> list: @@ -107,9 +106,9 @@ def core(session): session : sessionmaker A SQLAlchemy sessionmaker to use for database access. """ - with session() as session: + with session() as cur_session: d1maps = ( - session.query(DepthOneMapTable) + cur_session.query(DepthOneMapTable) .outerjoin( SkyCoverageTable, SkyCoverageTable.map_id == DepthOneMapTable.map_id ) @@ -118,9 +117,9 @@ def core(session): ) for d1map in d1maps: SkyCov = coverage_from_depthone(d1map) - session.add_all(SkyCov) + cur_session.add_all(SkyCov) - session.commit() + cur_session.commit() def main(): diff --git a/tests/test_act.py b/tests/test_act.py index cbad8ee..f566fce 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -7,9 +7,8 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -import mapcat.toolkit.act as act -import mapcat.toolkit.update_sky_coverage as update_sky_coverage from mapcat.database import DepthOneMapTable +from mapcat.toolkit import act, update_sky_coverage DATA_URLS = [ "https://g-0a470a.6b7bd8.0ec8.data.globus.org/act_dr6/dr6.02/depth1/depth1_maps/15056/depth1_1505603190_pa4_f150_info.hdf", @@ -125,8 +124,6 @@ def run_migration(database_path: str): alembic_cfg.set_main_option("sqlalchemy.url", database_url) command.upgrade(alembic_cfg, "head") - return - @pytest.fixture(scope="session", autouse=True) def database_sessionmaker(tmp_path_factory): diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index b95f198..04fd4a9 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -32,8 +32,6 @@ def run_migration(database_path: str): alembic_cfg.set_main_option("sqlalchemy.url", database_url) command.upgrade(alembic_cfg, "head") - return - @pytest.fixture(scope="module", autouse=True) def database_sessionmaker(tmp_path_factory): @@ -210,11 +208,10 @@ def test_create_depth_one(database_sessionmaker): assert sky.y == "2" # Check bad map ID raises ValueError - with pytest.raises(ValueError): - with database_sessionmaker() as session: - result = session.get(DepthOneMapTable, 999999) - if result is None: - raise ValueError("Map ID does not exist") + with pytest.raises(ValueError), database_sessionmaker() as session: + result = session.get(DepthOneMapTable, 999999) + if result is None: + raise ValueError("Map ID does not exist") def test_add_remove_child_tables(database_sessionmaker): @@ -306,33 +303,29 @@ def test_add_remove_child_tables(database_sessionmaker): session.delete(x) session.commit() - with pytest.raises(ValueError): - with database_sessionmaker() as session: - x = session.get(TimeDomainProcessingTable, proc_id) - if x is None: - raise ValueError("Not found") + with pytest.raises(ValueError), database_sessionmaker() as session: + x = session.get(TimeDomainProcessingTable, proc_id) + if x is None: + raise ValueError("Not found") - with pytest.raises(ValueError): - with database_sessionmaker() as session: - x = session.get(PointingResidualTable, point_id) - if x is None: - raise ValueError("Not found") + with pytest.raises(ValueError), database_sessionmaker() as session: + x = session.get(PointingResidualTable, point_id) + if x is None: + raise ValueError("Not found") with database_sessionmaker() as session: tod = session.get(TODDepthOneTable, tod_id) assert tod is not None - with pytest.raises(ValueError): - with database_sessionmaker() as session: - x = session.get(PipelineInformationTable, pipe_id) - if x is None: - raise ValueError("Not found") + with pytest.raises(ValueError), database_sessionmaker() as session: + x = session.get(PipelineInformationTable, pipe_id) + if x is None: + raise ValueError("Not found") - with pytest.raises(ValueError): - with database_sessionmaker() as session: - x = session.get(SkyCoverageTable, sky_id) - if x is None: - raise ValueError("Not found") + with pytest.raises(ValueError), database_sessionmaker() as session: + x = session.get(SkyCoverageTable, sky_id) + if x is None: + raise ValueError("Not found") def test_create_atomic_map_coadd(database_sessionmaker): @@ -502,11 +495,10 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert daily_parent_coadds[0].coadd_id == weekly_coadd_id # Check bad map ID raises ValueError - with pytest.raises(ValueError): - with database_sessionmaker() as session: - result = session.get(AtomicMapCoaddTable, 999999) - if result is None: - raise ValueError("Map ID does not exist") + with pytest.raises(ValueError), database_sessionmaker() as session: + result = session.get(AtomicMapCoaddTable, 999999) + if result is None: + raise ValueError("Map ID does not exist") def test_add_remove_atomic_map_coadd(database_sessionmaker): @@ -554,11 +546,10 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): session.delete(x) session.commit() - with pytest.raises(ValueError): - with database_sessionmaker() as session: - daily = session.get(AtomicMapCoaddTable, daily_coadd_id) - if len(daily.parent_coadds) == 0: - raise ValueError("Daily map has no parent coadds") + with pytest.raises(ValueError), database_sessionmaker() as session: + daily = session.get(AtomicMapCoaddTable, daily_coadd_id) + if len(daily.parent_coadds) == 0: + raise ValueError("Daily map has no parent coadds") # Check in reverse with database_sessionmaker() as session: @@ -585,8 +576,7 @@ def test_add_remove_atomic_map_coadd(database_sessionmaker): session.delete(x) session.commit() - with pytest.raises(ValueError): - with database_sessionmaker() as session: - weekly2 = session.get(AtomicMapCoaddTable, weekly2_coadd_id) - if len(weekly2.child_coadds) == 0: - raise ValueError("Weekly map has no child coadds") + with pytest.raises(ValueError), database_sessionmaker() as session: + weekly2 = session.get(AtomicMapCoaddTable, weekly2_coadd_id) + if len(weekly2.child_coadds) == 0: + raise ValueError("Weekly map has no child coadds") diff --git a/tests/test_mapmaking.py b/tests/test_mapmaking.py index 797825e..0cd44c8 100644 --- a/tests/test_mapmaking.py +++ b/tests/test_mapmaking.py @@ -20,8 +20,6 @@ def run_migration(database_path: str): alembic_cfg.set_main_option("sqlalchemy.url", database_url) command.upgrade(alembic_cfg, "head") - return - @pytest.fixture(scope="session", autouse=True) def database_sessionmaker(tmp_path_factory): From c4410f540b6b83832bdacfba488aaeba5855c96f Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 20 Feb 2026 14:21:36 -0500 Subject: [PATCH 25/35] temp: checking DEPTH_ONE_MAP path --- mapcat/toolkit/update_sky_coverage.py | 3 +++ tests/test_act.py | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index ec77a19..5ab3824 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -87,6 +87,9 @@ def coverage_from_depthone(d1table: DepthOneMapTable) -> list[SkyCoverageTable]: A list of sky coverage tiles that cover the map """ tmap_path = resolve_tmap(d1table) + print("\n\n\n") + print(tmap_path) + print("\n\n\n") tmap = enmap.read_map(str(tmap_path)) coverage_tiles = get_sky_coverage(tmap) diff --git a/tests/test_act.py b/tests/test_act.py index f566fce..e16cbb0 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -233,8 +233,6 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): with database_sessionmaker() as session: d1maps = session.query(DepthOneMapTable).all() for d1map in d1maps: - print("\n\n D1MAP: ", d1map) - print("\n\n") assert len(d1map.depth_one_sky_coverage) > 0 for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. From 34ec728ff4315da731f44f6b550a3c70a7a1b8d2 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 20 Feb 2026 14:32:09 -0500 Subject: [PATCH 26/35] temp: echo DEPTH_ONE_PATH --- .github/workflows/pytest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f0f536d..d431af7 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -30,6 +30,9 @@ jobs: - name: Sync uv run: uv pip install ".[dev]" + - name: Check DEPTH_ONE_PATH + run: echo $DEPTH_ONE_PATH + - name: Run tests run: pytest --cov From 38cbe26fb31bccbe4d42113262ba58301f47a4be Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Fri, 20 Feb 2026 14:37:14 -0500 Subject: [PATCH 27/35] fix: wrong variable name for DEPTH_ONE_PARENT --- .github/workflows/pytest.yml | 5 +---- mapcat/toolkit/update_sky_coverage.py | 3 --- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d431af7..048931a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,7 +3,7 @@ name: Run tests on: [push] env: - DEPTH_ONE_PATH: "../.pytest_cache/d1maps/" + DEPTH_ONE_PARENT: "../.pytest_cache/d1maps/" jobs: runtest: @@ -30,9 +30,6 @@ jobs: - name: Sync uv run: uv pip install ".[dev]" - - name: Check DEPTH_ONE_PATH - run: echo $DEPTH_ONE_PATH - - name: Run tests run: pytest --cov diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index 5ab3824..ec77a19 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -87,9 +87,6 @@ def coverage_from_depthone(d1table: DepthOneMapTable) -> list[SkyCoverageTable]: A list of sky coverage tiles that cover the map """ tmap_path = resolve_tmap(d1table) - print("\n\n\n") - print(tmap_path) - print("\n\n\n") tmap = enmap.read_map(str(tmap_path)) coverage_tiles = get_sky_coverage(tmap) From 2cf5e0fad352019693e64d8a9364406623644cf1 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Wed, 25 Mar 2026 15:07:49 -0400 Subject: [PATCH 28/35] feat: change dec to 0-17, ra to 0-35 and add plotting util --- mapcat/toolkit/plot_tiles.py | 73 +++++++++++++++++++++++++++ mapcat/toolkit/update_sky_coverage.py | 2 +- 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 mapcat/toolkit/plot_tiles.py diff --git a/mapcat/toolkit/plot_tiles.py b/mapcat/toolkit/plot_tiles.py new file mode 100644 index 0000000..4d951b0 --- /dev/null +++ b/mapcat/toolkit/plot_tiles.py @@ -0,0 +1,73 @@ +import matplotlib.pyplot as plt +import numpy as np +from pixell import enmap + +from mapcat.toolkit.update_sky_coverage import * + +imap_path = "/home/jack/act_planck_s08_s22_f150_daynight_map.fits" +imap = enmap.read_map(str(imap_path)) + +box = imap.box() + +dec_min, ra_max = np.rad2deg(box[0]) +dec_max, ra_min = np.rad2deg(box[1]) + +pad_low = int((90 + dec_min) * 6 * 2) +pad_high = int((90 - dec_max) * 6 * 2) + +imap = imap[0][::10, ::10] + +pad_map = np.pad( + imap, ((pad_low, pad_high), (0, 0)), mode="constant", constant_values=0 +) +del imap +plt.imshow(pad_map, vmin=-300, vmax=300, origin="lower") +plt.vlines( + np.arange(0, 360 * 6 * 2, 10 * 6 * 2), ymin=0, ymax=180 * 6 * 2, color="black", lw=1 +) +plt.hlines( + np.arange(0, 180 * 6 * 2, 10 * 6 * 2), xmin=0, xmax=360 * 6 * 2, color="black", lw=1 +) +plt.xticks(np.arange(0, 360 * 6 * 2, 20 * 6 * 2), labels=np.arange(0, 360, 20)) +plt.yticks(np.arange(0, 180 * 6 * 2, 10 * 6 * 2), labels=np.arange(-90, 90, 10)) +plt.xlabel("RA (degrees)") +plt.ylabel("Dec (degrees)") + +tmap_path = "/home/jack/dev/mapcat/.pytest_cache/d1maps/15056/depth1_1505603190_pa4_f150_map.fits" +tmap = enmap.read_map(str(tmap_path)) +coverage_tiles = get_sky_coverage(tmap) + +tbox = tmap.box() + +tdec_min, tra_max = np.rad2deg(tbox[0]) +tdec_max, tra_min = np.rad2deg(tbox[1]) + +tpad_low_dec = int((90 + tdec_min) * 6 * 2) +tpad_high_dec = int((90 - tdec_max) * 6 * 2) + +tpad_low_ra = int((180 + tra_min) * 6 * 2) +tpad_high_ra = int((180 - tra_max) * 6 * 2) + +tmap = tmap[0][::10, ::10] +tpad_map = np.pad( + tmap, + ((tpad_low_dec, tpad_high_dec), (tpad_low_ra, tpad_high_ra)), + mode="constant", + constant_values=0, +) + +plt.imshow(tpad_map, vmin=-300, vmax=300, origin="lower", alpha=0.5, cmap="seismic") + +for tile in coverage_tiles: + plt.gca().add_patch( + plt.Rectangle( + (tile[0] * 10 * 6 * 2, tile[1] * 10 * 6 * 2), + 10 * 6 * 2, + 10 * 6 * 2, + fill=False, + edgecolor="red", + lw=2, + ) + ) + +plt.savefig("/mnt/c/Users/Jack/Desktop/act_coverage.png", dpi=300) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index ec77a19..e6eb677 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -66,7 +66,7 @@ def get_sky_coverage(tmap: enmap.ndmap) -> list: ) submap = enmap.submap(tmap, skybox) if np.any(submap): - ra_idx.append(int(ra / 10)) + ra_idx.append(int(ra / 10) + 18) dec_id.append(int(dec / 10) + 9) return list(zip(ra_idx, dec_id)) From 1c129a8272e7d47b8c182f6852ed6d1de8de437c Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Wed, 25 Mar 2026 16:39:43 -0400 Subject: [PATCH 29/35] fix: update test to new ra/dec conventions and plot with ACT (e.g., r to l convention) for RA --- mapcat/toolkit/plot_tiles.py | 48 ++++++--- tests/test_act.py | 188 ++++++++++++++++------------------- 2 files changed, 119 insertions(+), 117 deletions(-) diff --git a/mapcat/toolkit/plot_tiles.py b/mapcat/toolkit/plot_tiles.py index 4d951b0..8123369 100644 --- a/mapcat/toolkit/plot_tiles.py +++ b/mapcat/toolkit/plot_tiles.py @@ -21,7 +21,15 @@ imap, ((pad_low, pad_high), (0, 0)), mode="constant", constant_values=0 ) del imap -plt.imshow(pad_map, vmin=-300, vmax=300, origin="lower") + +left_limit = pad_map.shape[1] +right_limit = 0 +top_limit = pad_map.shape[0] +bottom_limit = 0 +extent = [left_limit, right_limit, bottom_limit, top_limit] + + +plt.imshow(pad_map, vmin=-300, vmax=300, origin="lower", extent=extent) plt.vlines( np.arange(0, 360 * 6 * 2, 10 * 6 * 2), ymin=0, ymax=180 * 6 * 2, color="black", lw=1 ) @@ -33,30 +41,38 @@ plt.xlabel("RA (degrees)") plt.ylabel("Dec (degrees)") -tmap_path = "/home/jack/dev/mapcat/.pytest_cache/d1maps/15056/depth1_1505603190_pa4_f150_map.fits" -tmap = enmap.read_map(str(tmap_path)) -coverage_tiles = get_sky_coverage(tmap) +d1map_path = "/home/jack/dev/mapcat/.pytest_cache/d1maps/15056/depth1_1505603190_pa4_f150_map.fits" +d1map = enmap.read_map(str(d1map_path)) +coverage_tiles = get_sky_coverage(d1map) -tbox = tmap.box() +d1box = d1map.box() -tdec_min, tra_max = np.rad2deg(tbox[0]) -tdec_max, tra_min = np.rad2deg(tbox[1]) +d1dec_min, d1ra_max = np.rad2deg(d1box[0]) +d1dec_max, d1ra_min = np.rad2deg(d1box[1]) -tpad_low_dec = int((90 + tdec_min) * 6 * 2) -tpad_high_dec = int((90 - tdec_max) * 6 * 2) +d1pad_low_dec = int((90 + d1dec_min) * 6 * 2) +d1pad_high_dec = int((90 - d1dec_max) * 6 * 2) -tpad_low_ra = int((180 + tra_min) * 6 * 2) -tpad_high_ra = int((180 - tra_max) * 6 * 2) +d1pad_low_ra = int((180 + d1ra_min) * 6 * 2) +d1pad_high_ra = int((180 - d1ra_max) * 6 * 2) -tmap = tmap[0][::10, ::10] -tpad_map = np.pad( - tmap, - ((tpad_low_dec, tpad_high_dec), (tpad_low_ra, tpad_high_ra)), +d1map = d1map[0][::10, ::10] +d1pad_map = np.pad( + d1map, + ((d1pad_low_dec, d1pad_high_dec), (d1pad_high_ra, d1pad_low_ra)), mode="constant", constant_values=0, ) -plt.imshow(tpad_map, vmin=-300, vmax=300, origin="lower", alpha=0.5, cmap="seismic") +plt.imshow( + d1pad_map, + vmin=-300, + vmax=300, + origin="lower", + alpha=0.5, + cmap="seismic", + extent=extent, +) for tile in coverage_tiles: plt.gca().add_patch( diff --git a/tests/test_act.py b/tests/test_act.py index e16cbb0..270fbea 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -26,87 +26,87 @@ ] cov_mapping = { - "1505603190.0": [ - (-4, 3), - (-4, 4), - (-4, 6), - (-4, 7), - (-3, 3), - (-3, 4), - (-3, 5), - (-3, 6), - (-3, 7), - (-2, 3), - (-2, 4), - (-2, 5), - (-2, 6), - (-2, 7), - (-1, 3), - (-1, 4), - (-1, 5), - (-1, 6), - (-1, 7), - (0, 3), - (0, 4), - (0, 5), - (0, 6), - (0, 7), - (1, 3), - (1, 4), - (1, 5), - (1, 6), - (1, 7), - (2, 3), - (2, 4), - (2, 5), - (2, 6), - (2, 7), - (3, 3), - (3, 4), - (3, 5), - (3, 6), - (3, 7), - (4, 3), - (4, 4), - (4, 5), - (4, 6), - (4, 7), - (5, 3), - (5, 4), - (5, 5), - (5, 6), - (5, 7), - (6, 3), - (6, 4), - (6, 5), - (6, 6), - (6, 7), - (7, 3), - (7, 4), - (7, 5), - (7, 6), - (7, 7), - (8, 3), - (8, 4), - (8, 5), - (8, 6), - (8, 7), - (9, 3), - (9, 4), - (9, 5), - (9, 6), - (9, 7), - (10, 3), - (10, 4), - (10, 5), - (10, 6), - (10, 7), - (11, 3), - (11, 4), - (11, 5), - (11, 6), + "1505603190": [ + (14, 3), + (14, 4), + (14, 6), + (14, 7), + (15, 3), + (15, 4), + (15, 5), + (15, 6), + (15, 7), + (16, 3), + (16, 4), + (16, 5), + (16, 6), + (16, 7), + (17, 3), + (17, 4), + (17, 5), + (17, 6), + (17, 7), + (18, 3), + (18, 4), + (18, 5), + (18, 6), + (18, 7), + (19, 3), + (19, 4), + (19, 5), + (19, 6), + (19, 7), + (20, 3), + (20, 4), + (20, 5), + (20, 6), + (20, 7), + (21, 3), + (21, 4), + (21, 5), + (21, 6), + (21, 7), + (22, 3), + (22, 4), + (22, 5), + (22, 6), + (22, 7), + (23, 3), + (23, 4), + (23, 5), + (23, 6), + (23, 7), + (24, 3), + (24, 4), + (24, 5), + (24, 6), + (24, 7), + (25, 3), + (25, 4), + (25, 5), + (25, 6), + (25, 7), + (26, 3), + (26, 4), + (26, 5), + (26, 6), + (26, 7), + (27, 3), + (27, 4), + (27, 5), + (27, 6), + (27, 7), + (28, 3), + (28, 4), + (28, 5), + (28, 6), + (28, 7), + (29, 3), + (29, 4), + (29, 5), + (29, 6), ], - "1505646390.0": [(4, 3), (4, 4), (4, 5), (5, 3), (5, 4), (5, 5)], + "1505646390": [(22, 3), (22, 4), (22, 5), (23, 3), (23, 4), (23, 5)], } @@ -133,7 +133,7 @@ def database_sessionmaker(tmp_path_factory): tmp_path = tmp_path_factory.mktemp("mapcat") # Create a temporary SQLite database for testing. - database_path = tmp_path / "act_test.db" + database_path = tmp_path / "test.db" # Run the migration on the database. This is blocking. run_migration(database_path) @@ -155,7 +155,7 @@ def downloaded_data_file(request): Parameters ---------- - request : pytest.FixtureRequest + tmp_path_factory : pytest.TempPathFactory Factory to create temporary directory for downloaded files Returns ------- @@ -215,31 +215,17 @@ def test_act(database_sessionmaker, downloaded_data_file): assert map.frequency == "f150" assert map.ctime in [1505603190, 1505646390] - # Clean up, otherewise we interfere with test_sky_coverage - session.delete(map) - - session.commit() - - -def test_sky_coverage(database_sessionmaker, downloaded_data_file): - args = ap.Namespace( - glob="*/*_map.fits", - relative_to=downloaded_data_file, - telescope="act", - ) - act.core(session=database_sessionmaker, args=args) +def test_sky_coverage(database_sessionmaker): update_sky_coverage.core(session=database_sessionmaker) + with database_sessionmaker() as session: d1maps = session.query(DepthOneMapTable).all() + print("LEN", len(d1maps)) for d1map in d1maps: + print("\n\n\n\nD1MAP: ", d1map.depth_one_sky_coverage) assert len(d1map.depth_one_sky_coverage) > 0 for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. # The coverage tiles are stored in cov_mapping, which is a dict mapping from ctime to a list of (x,y) tuples representing the coverage tiles. - assert ( - int( - cov.x - ), # These should be ints, idk why I have to cast them (from str) - int(cov.y), - ) in cov_mapping[str(d1map.ctime)] + assert tuple(cov.x, cov.y) in cov_mapping[str(d1map.ctime)] From 13a27405f4945e1ca5cfe8034bfa58a1f995bb37 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Wed, 25 Mar 2026 16:58:02 -0400 Subject: [PATCH 30/35] fix: improper tuple construction --- tests/test_act.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_act.py b/tests/test_act.py index 270fbea..5e82b48 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -228,4 +228,4 @@ def test_sky_coverage(database_sessionmaker): for cov in d1map.depth_one_sky_coverage: # Shitty test to make sure the coverage tiles are correct, by checking against the known coverage for these two maps. # The coverage tiles are stored in cov_mapping, which is a dict mapping from ctime to a list of (x,y) tuples representing the coverage tiles. - assert tuple(cov.x, cov.y) in cov_mapping[str(d1map.ctime)] + assert (cov.x, cov.y) in cov_mapping[str(d1map.ctime)] From e2c71d1bb4e70b380246f1303991f833cfeb7d1a Mon Sep 17 00:00:00 2001 From: Josh Borrow Date: Thu, 26 Mar 2026 09:25:54 -0400 Subject: [PATCH 31/35] Add CLI script --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f701ceb..562b968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ [project.scripts] actingest = "mapcat.toolkit.act:main" mapcatmigrate = "mapcat.helper:migrate" +updatesky = "mapcat.toolkit.update_sky_coverage:main" [project.optional-dependencies] dev = [ From d4f23435bdd1a6bdde0ac73baf324047fa62ae69 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 26 Mar 2026 09:40:44 -0400 Subject: [PATCH 32/35] fix: updating coverage tests to new ra/dec conventions --- tests/test_act.py | 158 +++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/tests/test_act.py b/tests/test_act.py index e16cbb0..2067024 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -27,86 +27,86 @@ cov_mapping = { "1505603190.0": [ - (-4, 3), - (-4, 4), - (-4, 6), - (-4, 7), - (-3, 3), - (-3, 4), - (-3, 5), - (-3, 6), - (-3, 7), - (-2, 3), - (-2, 4), - (-2, 5), - (-2, 6), - (-2, 7), - (-1, 3), - (-1, 4), - (-1, 5), - (-1, 6), - (-1, 7), - (0, 3), - (0, 4), - (0, 5), - (0, 6), - (0, 7), - (1, 3), - (1, 4), - (1, 5), - (1, 6), - (1, 7), - (2, 3), - (2, 4), - (2, 5), - (2, 6), - (2, 7), - (3, 3), - (3, 4), - (3, 5), - (3, 6), - (3, 7), - (4, 3), - (4, 4), - (4, 5), - (4, 6), - (4, 7), - (5, 3), - (5, 4), - (5, 5), - (5, 6), - (5, 7), - (6, 3), - (6, 4), - (6, 5), - (6, 6), - (6, 7), - (7, 3), - (7, 4), - (7, 5), - (7, 6), - (7, 7), - (8, 3), - (8, 4), - (8, 5), - (8, 6), - (8, 7), - (9, 3), - (9, 4), - (9, 5), - (9, 6), - (9, 7), - (10, 3), - (10, 4), - (10, 5), - (10, 6), - (10, 7), - (11, 3), - (11, 4), - (11, 5), - (11, 6), + (14, 3), + (14, 4), + (14, 6), + (14, 7), + (15, 3), + (15, 4), + (15, 5), + (15, 6), + (15, 7), + (16, 3), + (16, 4), + (16, 5), + (16, 6), + (16, 7), + (17, 3), + (17, 4), + (17, 5), + (17, 6), + (17, 7), + (18, 3), + (18, 4), + (18, 5), + (18, 6), + (18, 7), + (19, 3), + (19, 4), + (19, 5), + (19, 6), + (19, 7), + (20, 3), + (20, 4), + (20, 5), + (20, 6), + (20, 7), + (21, 3), + (21, 4), + (21, 5), + (21, 6), + (21, 7), + (22, 3), + (22, 4), + (22, 5), + (22, 6), + (22, 7), + (23, 3), + (23, 4), + (23, 5), + (23, 6), + (23, 7), + (24, 3), + (24, 4), + (24, 5), + (24, 6), + (24, 7), + (25, 3), + (25, 4), + (25, 5), + (25, 6), + (25, 7), + (26, 3), + (26, 4), + (26, 5), + (26, 6), + (26, 7), + (27, 3), + (27, 4), + (27, 5), + (27, 6), + (27, 7), + (28, 3), + (28, 4), + (28, 5), + (28, 6), + (28, 7), + (29, 3), + (29, 4), + (29, 5), + (29, 6), ], - "1505646390.0": [(4, 3), (4, 4), (4, 5), (5, 3), (5, 4), (5, 5)], + "1505646390.0": [(22, 3), (22, 4), (22, 5), (23, 3), (23, 4), (23, 5)], } From 969985726ab797d654ca7158560804ffe14c7207 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Thu, 26 Mar 2026 15:22:52 -0400 Subject: [PATCH 33/35] fix: clean up some tests and increase coverage --- .gitignore | 1 + mapcat/core/__init__.py | 5 ++ mapcat/core/core.py | 52 +++++++++++++ mapcat/database/atomic_coadd.py | 2 +- mapcat/database/atomic_map.py | 2 +- mapcat/database/depth_one_map.py | 2 +- mapcat/helper.py | 2 +- mapcat/toolkit/reset.py | 6 +- mapcat/toolkit/update_sky_coverage.py | 101 +++++++++++++++++++++++--- tests/test_act.py | 62 ++++++++++++++++ tests/test_reset.py | 4 +- 11 files changed, 218 insertions(+), 21 deletions(-) create mode 100644 mapcat/core/__init__.py create mode 100644 mapcat/core/core.py diff --git a/.gitignore b/.gitignore index 54ee18a..db407ce 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist/* *.egg-info *.fits *.hdf +.coverage diff --git a/mapcat/core/__init__.py b/mapcat/core/__init__.py new file mode 100644 index 0000000..3bee488 --- /dev/null +++ b/mapcat/core/__init__.py @@ -0,0 +1,5 @@ +from .core import get_maps_by_coverage + +__all__ = [ + "get_maps_by_coverage", +] diff --git a/mapcat/core/core.py b/mapcat/core/core.py new file mode 100644 index 0000000..e324388 --- /dev/null +++ b/mapcat/core/core.py @@ -0,0 +1,52 @@ +from astropy.coordinates import ICRS +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mapcat.database import DepthOneMapTable +from mapcat.toolkit.update_sky_coverage import dec_to_index, ra_to_index + + +def get_maps_by_coverage( + position: ICRS, + session: Session, +) -> list[DepthOneMapTable]: + """ + Get the depth one maps that cover a given position. + + Parameters + ---------- + position : ICRS + The position to query for coverage. Should be in ICRS coordinates. + session : Session + The database session to use for the query. + + Returns + ------- + session.execute(stmt).scalars().all() : list[DepthOneMapTable] + A list of depth one maps that cover the given position. + + Raises + ------ + ValueError + If the RA or Dec of the position is out of bounds. + """ + ra = position.ra.deg + dec = position.dec.deg + + # These aren't covered since ICRS automatically wraps + # values back aground to 0-360 for RA and -90 to 90 for Dec. + if ra < 0 or ra > 360: # pragma: no cover + raise ValueError("RA must be between 0 and 360 degrees") + if dec < -90 or dec > 90: # pragma: no cover + raise ValueError("Dec must be between -90 and 90 degrees") + + ra_idx = ra_to_index(ra) + dec_idx = dec_to_index(dec) + + stmt = ( + select(DepthOneMapTable) + .join(DepthOneMapTable.depth_one_sky_coverage) + .filter_by(x=ra_idx, y=dec_idx) + ) + + return session.execute(stmt).scalars().all() diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index fad6a61..73653e9 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from .atomic_map import AtomicMapTable -from .links import AtomicMapToCoaddTable, CoaddMapToCoaddTable +from .links import AtomicMapToCoaddTable, CoaddMapToCoaddTable # pragma: no cover class AtomicMapCoaddTable(SQLModel, table=True): diff --git a/mapcat/database/atomic_map.py b/mapcat/database/atomic_map.py index 57304a7..08753a3 100644 --- a/mapcat/database/atomic_map.py +++ b/mapcat/database/atomic_map.py @@ -9,7 +9,7 @@ from .links import AtomicMapToCoaddTable if TYPE_CHECKING: - from .atomic_coadd import AtomicMapCoaddTable + from .atomic_coadd import AtomicMapCoaddTable # pragma: no cover class AtomicMapTable(SQLModel, table=True): diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py index aa32033..bb8f7f6 100644 --- a/mapcat/database/depth_one_map.py +++ b/mapcat/database/depth_one_map.py @@ -6,7 +6,7 @@ from sqlmodel import JSON, Field, Relationship, SQLModel -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from .depth_one_coadd import DepthOneCoaddTable from .pipeline_information import PipelineInformationTable from .pointing_residual import PointingResidualTable diff --git a/mapcat/helper.py b/mapcat/helper.py index 6efc7d6..29d348a 100644 --- a/mapcat/helper.py +++ b/mapcat/helper.py @@ -66,7 +66,7 @@ def session(self) -> sessionmaker: settings = Settings() -def migrate(): +def migrate(): # pragma: no cover """ CLI script to run 'alembic upgrade head' with the correct location. """ diff --git a/mapcat/toolkit/reset.py b/mapcat/toolkit/reset.py index 8cdcc6d..d055bd5 100644 --- a/mapcat/toolkit/reset.py +++ b/mapcat/toolkit/reset.py @@ -123,8 +123,7 @@ def main(): type=float, default=None, help=( - "Only reset entries for maps whose ctime is >= START_TIME " - "(unix timestamp)." + "Only reset entries for maps whose ctime is >= START_TIME (unix timestamp)." ), ) @@ -133,8 +132,7 @@ def main(): type=float, default=None, help=( - "Only reset entries for maps whose ctime is <= END_TIME " - "(unix timestamp)." + "Only reset entries for maps whose ctime is <= END_TIME (unix timestamp)." ), ) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index e6eb677..cecb1ec 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -25,6 +25,88 @@ def resolve_tmap(d1table: DepthOneMapTable) -> Path: return settings.depth_one_parent / d1table.mean_time_path +def index_to_skybox(ra_idx: int, dec_idx: int) -> np.ndarray: + """ + Convert a sky coverage tile index to a sky box in radians + + Parameters + ---------- + ra_idx : int + The RA index of the sky coverage tile + dec_idx : int + The Dec index of the sky coverage tile + + Returns + ------- + skybox : np.ndarray + A 2x2 array containing the corners of the sky box in radians, in the format [[dec_min, ra_max], [dec_max, ra_min]] + """ + ra_min = (ra_idx - 18) * 10 + ra_max = ra_min + 10 + dec_min = (dec_idx - 9) * 10 + dec_max = dec_min + 10 + + return np.array( + [ + [np.deg2rad(dec_min), np.deg2rad(ra_max)], + [np.deg2rad(dec_max), np.deg2rad(ra_min)], + ] + ) + + +def ra_to_index(ra: float) -> int: + """ + Convert an ra in degrees to a sky coverage tile index + + Parameters + ---------- + ra : float + The ra in degrees to convert + + Returns + ------- + idx : int + The sky coverage tile index corresponding to the input ra + """ + return int(np.floor(ra / 10)) + + +def dec_to_index(dec: float) -> int: + """ + Convert a dec in degrees to a sky coverage tile index + + Parameters + ---------- + dec : float + The dec in degrees to convert + + Returns + ------- + idx : int + The sky coverage tile index corresponding to the input dec + """ + return int(np.floor(dec / 10)) + 9 + + +def _ra_to_index_pixell(ra: float) -> int: + """ + Convert an ra in degrees to a sky coverage tile index using the + pixell convention where -180 < ra < 180. You should probably + not ever touch this function. + + Parameters + ---------- + ra : float + The ra in degrees to convert + + Returns + ------- + idx : int + The sky coverage tile index corresponding to the input ra + """ + return int(np.floor(ra / 10)) + 18 + + def get_sky_coverage(tmap: enmap.ndmap) -> list: """ Given the time map of a depth1 map, return the list @@ -49,27 +131,26 @@ def get_sky_coverage(tmap: enmap.ndmap) -> list: dec_max = np.ceil(dec_max / 10) * 10 ra_min = np.floor(ra_min / 10) * 10 ra_max = np.ceil(ra_max / 10) * 10 + ra_min += 180 # Convert from pixel standard to normal RA convention + ra_max += 180 ras = np.arange(ra_min, ra_max, 10) decs = np.arange(dec_min, dec_max, 10) ra_idx = [] - dec_id = [] + dec_idx = [] for ra in ras: for dec in decs: - skybox = np.array( - [ - [np.deg2rad(dec), np.deg2rad(ra + 10)], - [np.deg2rad(dec + 10), np.deg2rad(ra)], - ] - ) + ra_id = ra_to_index(ra) + dec_id = dec_to_index(dec) + skybox = index_to_skybox(ra_id, dec_id) submap = enmap.submap(tmap, skybox) if np.any(submap): - ra_idx.append(int(ra / 10) + 18) - dec_id.append(int(dec / 10) + 9) + ra_idx.append(ra_id) + dec_idx.append(dec_id) - return list(zip(ra_idx, dec_id)) + return list(zip(ra_idx, dec_idx)) def coverage_from_depthone(d1table: DepthOneMapTable) -> list[SkyCoverageTable]: diff --git a/tests/test_act.py b/tests/test_act.py index 2067024..a89cba5 100644 --- a/tests/test_act.py +++ b/tests/test_act.py @@ -2,11 +2,16 @@ import os from pathlib import Path +import astropy.units as u +import numpy as np import pytest import requests +from astropy.coordinates import ICRS +from pixell import enmap from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from mapcat.core import get_maps_by_coverage from mapcat.database import DepthOneMapTable from mapcat.toolkit import act, update_sky_coverage @@ -222,6 +227,7 @@ def test_act(database_sessionmaker, downloaded_data_file): def test_sky_coverage(database_sessionmaker, downloaded_data_file): + # I'm not sure we need this test any more, test_sky_coverage_2 is better args = ap.Namespace( glob="*/*_map.fits", relative_to=downloaded_data_file, @@ -243,3 +249,59 @@ def test_sky_coverage(database_sessionmaker, downloaded_data_file): ), # These should be ints, idk why I have to cast them (from str) int(cov.y), ) in cov_mapping[str(d1map.ctime)] + + with database_sessionmaker() as session: + maps = session.query(DepthOneMapTable).all() + for map in maps: + # Clean up + session.delete(map) + + session.commit() + + +def test_sky_coverage_2(database_sessionmaker, downloaded_data_file): + args = ap.Namespace( + glob="*/*_map.fits", + relative_to=downloaded_data_file, + telescope="act", + ) + + act.core(session=database_sessionmaker, args=args) + + update_sky_coverage.core(session=database_sessionmaker) + + d1maps = act.glob(args.glob, args.relative_to, args.telescope) + with database_sessionmaker() as session: + for d1map in d1maps: + cur_map = enmap.read_map(str(downloaded_data_file) + "/" + d1map.map_path) + nonzero_radec = cur_map.pix2sky(np.where(cur_map[0] != 0)) + idx = np.linspace(0, len(nonzero_radec) - 1, 1000) + idx = np.round(idx).astype(int) + nonzero_radec = nonzero_radec.T[ + idx + ] # Only test a subset of the nonzero pixels to speed up the test + for pix in nonzero_radec: + coord = ICRS( + (pix[1] + np.pi) * u.rad, pix[0] * u.rad + ) # Convert from pixel standard to normal RA convention + return_d1map = get_maps_by_coverage(coord, session) + assert d1map.map_name in [m.map_name for m in return_d1map] + + ra, dec = 0, 0 # Test a point outside the coverage, should return an empty list + coord = ICRS(ra * u.rad, dec * u.rad) + return_d1map = get_maps_by_coverage(coord, session) + assert len(return_d1map) == 0 + + with database_sessionmaker() as session: + maps = session.query(DepthOneMapTable).all() + for map in maps: + # Clean up + session.delete(map) + + session.commit() + + +def test_ra_to_index_pixell(): + assert update_sky_coverage._ra_to_index_pixell(-180) == 0 + assert update_sky_coverage._ra_to_index_pixell(170) == 35 + assert update_sky_coverage._ra_to_index_pixell(0) == 18 diff --git a/tests/test_reset.py b/tests/test_reset.py index a2fbcb1..2650631 100644 --- a/tests/test_reset.py +++ b/tests/test_reset.py @@ -19,9 +19,7 @@ def run_migration(database_path: str): from alembic import command from alembic.config import Config - alembic_cfg = Config( - str(Path(__file__).parent.parent / "mapcat" / "alembic.ini") - ) + alembic_cfg = Config(str(Path(__file__).parent.parent / "mapcat" / "alembic.ini")) database_url = f"sqlite:///{database_path}" alembic_cfg.set_main_option("sqlalchemy.url", database_url) command.upgrade(alembic_cfg, "head") From 115cefca2d9555974be46900286022af864d05ed Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Wed, 1 Apr 2026 14:05:28 -0400 Subject: [PATCH 34/35] fix: making index_to_skybox consistent with ra_to_index and changing get_sky_coverage to account for pixell convention --- mapcat/toolkit/update_sky_coverage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py index cecb1ec..3679ad0 100644 --- a/mapcat/toolkit/update_sky_coverage.py +++ b/mapcat/toolkit/update_sky_coverage.py @@ -41,7 +41,7 @@ def index_to_skybox(ra_idx: int, dec_idx: int) -> np.ndarray: skybox : np.ndarray A 2x2 array containing the corners of the sky box in radians, in the format [[dec_min, ra_max], [dec_max, ra_min]] """ - ra_min = (ra_idx - 18) * 10 + ra_min = ra_idx * 10 ra_max = ra_min + 10 dec_min = (dec_idx - 9) * 10 dec_max = dec_min + 10 @@ -145,6 +145,7 @@ def get_sky_coverage(tmap: enmap.ndmap) -> list: ra_id = ra_to_index(ra) dec_id = dec_to_index(dec) skybox = index_to_skybox(ra_id, dec_id) + skybox[..., 1] -= np.pi # Convert from standard RA to pixell convention submap = enmap.submap(tmap, skybox) if np.any(submap): ra_idx.append(ra_id) From ac9b30d544cfbef9dbdf08d37a9f55719bb321b4 Mon Sep 17 00:00:00 2001 From: John Orlowski-Scherer Date: Wed, 1 Apr 2026 14:58:06 -0400 Subject: [PATCH 35/35] feat: update plot tiles to use argparse --- mapcat/toolkit/plot_tiles.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mapcat/toolkit/plot_tiles.py b/mapcat/toolkit/plot_tiles.py index 8123369..71467e2 100644 --- a/mapcat/toolkit/plot_tiles.py +++ b/mapcat/toolkit/plot_tiles.py @@ -1,10 +1,21 @@ +import argparse as ap + import matplotlib.pyplot as plt import numpy as np from pixell import enmap from mapcat.toolkit.update_sky_coverage import * -imap_path = "/home/jack/act_planck_s08_s22_f150_daynight_map.fits" +parser = ap.ArgumentParser() +parser.add_argument("--imap_path", type=str) +parser.add_argument("--d1map_path", type=str) +parser.add_argument("--opath", type=str) + +args = parser.parse_args() +imap_path = args.imap_path +d1map_path = args.d1map_path +opath = args.opath + imap = enmap.read_map(str(imap_path)) box = imap.box() @@ -41,7 +52,6 @@ plt.xlabel("RA (degrees)") plt.ylabel("Dec (degrees)") -d1map_path = "/home/jack/dev/mapcat/.pytest_cache/d1maps/15056/depth1_1505603190_pa4_f150_map.fits" d1map = enmap.read_map(str(d1map_path)) coverage_tiles = get_sky_coverage(d1map) @@ -86,4 +96,4 @@ ) ) -plt.savefig("/mnt/c/Users/Jack/Desktop/act_coverage.png", dpi=300) +plt.savefig(opath + "act_coverage.png", dpi=300)