From 5e4434454b529592797ee081eeafd8e86bfe24be Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:59:07 +0100 Subject: [PATCH 01/10] feat(skills): fetch a window per point and name blocked hosts Weather scripts take --date-col with --days, so each point is read from its own date, such as a planting date, instead of one period for all points. Every script checks its store first and fails with the blocked host named. The HTTP stores honour HTTPS_PROXY. Points with missing columns fail clearly. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01LpE9sWSPBk27r6awN4xP7W --- skills/agera5-climate/scripts/fetch.py | 100 ++++++++++++++++++--- skills/chirps-rainfall/scripts/fetch.py | 88 ++++++++++++++++-- skills/chirts-temperature/scripts/fetch.py | 93 +++++++++++++++++-- skills/elevation/scripts/fetch.py | 21 +++++ skills/soilgrids-soil/scripts/fetch.py | 21 +++++ 5 files changed, 301 insertions(+), 22 deletions(-) diff --git a/skills/agera5-climate/scripts/fetch.py b/skills/agera5-climate/scripts/fetch.py index ec5168b..622fa75 100644 --- a/skills/agera5-climate/scripts/fetch.py +++ b/skills/agera5-climate/scripts/fetch.py @@ -5,12 +5,16 @@ """Daily AgERA5 weather for points. One row per point and day. uv run fetch.py points.csv weather.parquet --start 2020-01-01 [--end 2025-12-31] [--vars srad,rhmax,rhmin,wind,et0,vpd] + uv run fetch.py points.csv weather.parquet --date-col planting_date --days 100 points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. Needs a Copernicus CDS key in CDSAPI_KEY or ~/.cdsapirc. +With --date-col each point gets its own window, from its date (day 0) to --days later. Default variables are the ones CHIRPS and CHIRTS do not have; the rest are available by name.""" import argparse import os +import urllib.error +import urllib.request from pathlib import Path import dask @@ -18,7 +22,9 @@ import xarray as xr STORE = "https://arco.datastores.ecmwf.int/cadl-arco-geo-001/arco/sis_agrometeorological_indicators/all/geoChunked.zarr" +HOSTS = ["arco.datastores.ecmwf.int"] # what a firewall must allow THREADS = 32 # reads wait on the network, not the CPU +TILE = 3 # degrees; close to the store's spatial chunk, so grouped reads rarely fetch a chunk twice # friendly name -> (store variable, scale, offset) so that value = store * scale + offset VARS = { "srad": ("Solar_Radiation_Flux", 1e-6, 0), # J m-2 d-1 -> MJ m-2 d-1 @@ -47,9 +53,32 @@ def cds_key() -> str: raise SystemExit("No CDS key. Set CDSAPI_KEY or write ~/.cdsapirc; see SKILL.md for how to get one.") +def reachable(hosts: list[str]) -> None: + """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" + for host in hosts: + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + pass # the host answered + except OSError as exc: + raise ConnectionError( + f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " + "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." + ) from None + + +def open_store() -> xr.Dataset: + # trust_env makes the HTTP client honour HTTPS_PROXY, which sandboxes and institutional networks route through + options = {"headers": {"Authorization": f"Bearer {cds_key()}"}, "client_kwargs": {"trust_env": True}} + return xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options=options) + + def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -76,14 +105,45 @@ def select_daily(ds, points, start, end, lat, lon): return sub.reindex(pt=points.index) -def fetch(points: pd.DataFrame, start: str, end: str | None = None, names: list[str] | None = None) -> pd.DataFrame: - points = validate_points(points) - names = DEFAULT.split(",") if names is None else names - if bad := set(names) - set(VARS): - raise ValueError(f"unknown variables {sorted(bad)}; choose from {', '.join(VARS)}") - ds = xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"headers": {"Authorization": f"Bearer {cds_key()}"}}) - store_vars = [VARS[n][0] for n in names] - sub = select_daily(ds[store_vars], points, start, end, "latitude", "longitude") +def windows(points: pd.DataFrame, days: int): + """Yield (points, start, end) reads that cover each point's own window, its date to `days` later. + + Points in one spatial chunk whose windows overlap share a read, so no chunk is fetched for dates nobody asked for.""" + if "date" not in points or days < 0: + raise ValueError("a window per point needs a date column and days of 0 or more") + pts = points.assign(date=pd.to_datetime(points.date).dt.normalize()) + pts = pts[pts.date.notna()].sort_values("date") # points without a date have no window + if pts.empty: + raise ValueError("no point has a date") + span = pd.Timedelta(days=days) + for _, tile in pts.groupby([(pts[c] // TILE).fillna(-999) for c in ("lat", "lon")]): + run = (tile.date > (tile.date + span).cummax().shift()).cumsum() # a new read where windows stop overlapping + for _, g in tile.groupby(run): + yield g.reset_index(drop=True), g.date.min(), g.date.max() + span + + +def clip(parts: list[pd.DataFrame], points: pd.DataFrame, days: int) -> pd.DataFrame: + """Keep each point's own window and count its days, 0 on the point's date.""" + df = pd.concat(parts, ignore_index=True) + day0 = pd.to_datetime(points.set_index("id").date).dt.normalize() + df.insert(2, "day", (pd.to_datetime(df.date) - df.id.map(day0)).dt.days) + return df[df.day.between(0, days)].sort_values(["id", "date"]).reset_index(drop=True) + + +def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: + """One period for all points (--start), or a window per point (--date-col with --days).""" + if bool(a.start) == bool(a.date_col) or (a.date_col and (a.days is None or a.end)): + p.error("give --start [--end] for one period, or --date-col with --days for a window per point") + points = pd.read_csv(a.points, dtype={"id": str}) + if a.date_col and a.date_col != "date": + if a.date_col not in points: + p.error(f"no column {a.date_col} in {a.points}") + points = points.drop(columns="date", errors="ignore").rename(columns={a.date_col: "date"}) + return points + + +def daily(ds: xr.Dataset, points: pd.DataFrame, start, end, names: list[str]) -> pd.DataFrame: + sub = select_daily(ds[[VARS[n][0] for n in names]], points, start, end, "latitude", "longitude") df = sub.to_dataframe().reset_index().rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) for n in names: var, scale, offset = VARS[n] @@ -93,15 +153,35 @@ def fetch(points: pd.DataFrame, start: str, end: str | None = None, names: list[ return df[["id", "date", *names, "cell_lat", "cell_lon"]] +def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, names: list[str] | None = None, days: int | None = None) -> pd.DataFrame: + """One period for all points (start, end), or with days a window per point from its `date` column.""" + points = validate_points(points) + names = DEFAULT.split(",") if names is None else names + if bad := set(names) - set(VARS): + raise ValueError(f"unknown variables {sorted(bad)}; choose from {', '.join(VARS)}") + if days is None and start is None: + raise ValueError("give start, or days with a date column in points") + reachable(HOSTS) + ds = open_store() + if days is None: + return daily(ds, points, start, end, names) + return clip([daily(ds, g, s, e, names) for g, s, e in windows(points, days)], points, days) + + if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("points"), p.add_argument("out"), p.add_argument("--start", required=True), p.add_argument("--end") + p.add_argument("points"), p.add_argument("out"), p.add_argument("--start"), p.add_argument("--end") + p.add_argument("--date-col", help="column with one date per point, such as the planting date") + p.add_argument("--days", type=int, help="days to fetch after each point's date; 100 gives days 0 to 100") p.add_argument("--vars", default=DEFAULT, help=f"comma-separated from {', '.join(VARS)}") a = p.parse_args() names = a.vars.split(",") - out = fetch(pd.read_csv(a.points, dtype={"id": str}), a.start, a.end, names) + points = cli_points(p, a) + out = fetch(points, a.start, a.end, names, a.days) out.to_csv(a.out, index=False) if a.out.endswith(".csv") else out.to_parquet(a.out, index=False, compression="zstd") print(f"{out.id.nunique()} points, {out.date.min()} to {out.date.max()}, {len(out):,} rows, {names} -> {a.out}") miss = out[names].isna().all(axis=1).groupby(out.id).all() if miss.any(): print(f"all-NaN: {list(miss[miss].index)}") + if nodate := sorted(set(points.id) - set(out.id)): + print(f"no date, so no window fetched: {nodate}") diff --git a/skills/chirps-rainfall/scripts/fetch.py b/skills/chirps-rainfall/scripts/fetch.py index 718e174..4a24228 100644 --- a/skills/chirps-rainfall/scripts/fetch.py +++ b/skills/chirps-rainfall/scripts/fetch.py @@ -5,17 +5,38 @@ """Daily CHIRPS v3 rainfall for points. One row per point and day, mm. uv run fetch.py points.csv out.parquet --start 2020-01-01 [--end 2025-12-31] + uv run fetch.py points.csv out.parquet --date-col planting_date --days 100 -points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store.""" +points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store. +With --date-col each point gets its own window, from its date (day 0) to --days later.""" import argparse +import urllib.error +import urllib.request + import dask import icechunk import pandas as pd import xarray as xr STORE = ("dynamical-ucsb-chc-chirps", "ucsb-chc-chirps-analysis-final/v0.1.0.icechunk") +HOSTS = ["dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com"] # what a firewall must allow THREADS = 32 # reads wait on the network, not the CPU +TILE = 3 # degrees; the store's spatial chunk, so grouped reads never fetch a chunk twice + + +def reachable(hosts: list[str]) -> None: + """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" + for host in hosts: + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + pass # the host answered + except OSError as exc: + raise ConnectionError( + f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " + "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." + ) from None def open_store() -> xr.Dataset: @@ -26,7 +47,10 @@ def open_store() -> xr.Dataset: def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -53,9 +77,44 @@ def select_daily(ds, points, start, end, lat, lon): return sub.reindex(pt=points.index) -def fetch(points: pd.DataFrame, start: str, end: str | None = None) -> pd.DataFrame: - points = validate_points(points) - ds = open_store() +def windows(points: pd.DataFrame, days: int): + """Yield (points, start, end) reads that cover each point's own window, its date to `days` later. + + Points in one spatial chunk whose windows overlap share a read, so no chunk is fetched for dates nobody asked for.""" + if "date" not in points or days < 0: + raise ValueError("a window per point needs a date column and days of 0 or more") + pts = points.assign(date=pd.to_datetime(points.date).dt.normalize()) + pts = pts[pts.date.notna()].sort_values("date") # points without a date have no window + if pts.empty: + raise ValueError("no point has a date") + span = pd.Timedelta(days=days) + for _, tile in pts.groupby([(pts[c] // TILE).fillna(-999) for c in ("lat", "lon")]): + run = (tile.date > (tile.date + span).cummax().shift()).cumsum() # a new read where windows stop overlapping + for _, g in tile.groupby(run): + yield g.reset_index(drop=True), g.date.min(), g.date.max() + span + + +def clip(parts: list[pd.DataFrame], points: pd.DataFrame, days: int) -> pd.DataFrame: + """Keep each point's own window and count its days, 0 on the point's date.""" + df = pd.concat(parts, ignore_index=True) + day0 = pd.to_datetime(points.set_index("id").date).dt.normalize() + df.insert(2, "day", (pd.to_datetime(df.date) - df.id.map(day0)).dt.days) + return df[df.day.between(0, days)].sort_values(["id", "date"]).reset_index(drop=True) + + +def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: + """One period for all points (--start), or a window per point (--date-col with --days).""" + if bool(a.start) == bool(a.date_col) or (a.date_col and (a.days is None or a.end)): + p.error("give --start [--end] for one period, or --date-col with --days for a window per point") + points = pd.read_csv(a.points, dtype={"id": str}) + if a.date_col and a.date_col != "date": + if a.date_col not in points: + p.error(f"no column {a.date_col} in {a.points}") + points = points.drop(columns="date", errors="ignore").rename(columns={a.date_col: "date"}) + return points + + +def daily(ds: xr.Dataset, points: pd.DataFrame, start, end) -> pd.DataFrame: sub = select_daily(ds.precipitation_surface, points, start, end, "latitude", "longitude") df = (sub * 86400).to_dataframe(name="precip_mm").reset_index() # kg m-2 s-1 -> mm/day df["id"] = points.id.values[df.pt] @@ -64,13 +123,30 @@ def fetch(points: pd.DataFrame, start: str, end: str | None = None) -> pd.DataFr return df[["id", "date", "precip_mm", "cell_lat", "cell_lon"]].astype({"precip_mm": "float32"}) +def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, days: int | None = None) -> pd.DataFrame: + """One period for all points (start, end), or with days a window per point from its `date` column.""" + points = validate_points(points) + if days is None and start is None: + raise ValueError("give start, or days with a date column in points") + reachable(HOSTS) + ds = open_store() + if days is None: + return daily(ds, points, start, end) + return clip([daily(ds, g, s, e) for g, s, e in windows(points, days)], points, days) + + if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("points"), p.add_argument("out"), p.add_argument("--start", required=True), p.add_argument("--end") + p.add_argument("points"), p.add_argument("out"), p.add_argument("--start"), p.add_argument("--end") + p.add_argument("--date-col", help="column with one date per point, such as the planting date") + p.add_argument("--days", type=int, help="days to fetch after each point's date; 100 gives days 0 to 100") a = p.parse_args() - out = fetch(pd.read_csv(a.points, dtype={"id": str}), a.start, a.end) + points = cli_points(p, a) + out = fetch(points, a.start, a.end, a.days) out.to_csv(a.out, index=False) if a.out.endswith(".csv") else out.to_parquet(a.out, index=False, compression="zstd") sea = out.groupby("id").precip_mm.apply(lambda s: s.isna().all()) print(f"{out.id.nunique()} points, {out.date.min()} to {out.date.max()}, {len(out):,} rows -> {a.out}") if sea.any(): print(f"all-NaN (missing coordinates, outside coverage or source nodata): {list(sea[sea].index)}") + if nodate := sorted(set(points.id) - set(out.id)): + print(f"no date, so no window fetched: {nodate}") diff --git a/skills/chirts-temperature/scripts/fetch.py b/skills/chirts-temperature/scripts/fetch.py index 3fd8a46..5816246 100644 --- a/skills/chirts-temperature/scripts/fetch.py +++ b/skills/chirts-temperature/scripts/fetch.py @@ -5,22 +5,51 @@ """Daily CHIRTS-ERA5 Tmax and Tmin for points. One row per point and day, degrees C. uv run fetch.py points.csv out.parquet --start 2020-01-01 [--end 2025-12-31] + uv run fetch.py points.csv out.parquet --date-col planting_date --days 100 -points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store.""" +points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store. +With --date-col each point gets its own window, from its date (day 0) to --days later.""" import argparse +import urllib.error +import urllib.request + import dask import pandas as pd import xarray as xr STORE = "https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr" +HOSTS = ["huggingface.co"] # what a firewall must allow, plus the download host it redirects to (scripts/check.py names it) THREADS = 32 # reads wait on the network, not the CPU +TILE = 15 # degrees; the store's spatial chunk, so grouped reads never fetch a chunk twice TMAX_OK = 60 # a few source pixels hold impossible values; mask above this +def reachable(hosts: list[str]) -> None: + """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" + for host in hosts: + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + pass # the host answered + except OSError as exc: + raise ConnectionError( + f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " + "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." + ) from None + + +def open_store() -> xr.Dataset: + # trust_env makes the HTTP client honour HTTPS_PROXY, which sandboxes and institutional networks route through + return xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"client_kwargs": {"trust_env": True}}) + + def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -47,9 +76,44 @@ def select_daily(ds, points, start, end, lat, lon): return sub.reindex(pt=points.index) -def fetch(points: pd.DataFrame, start: str, end: str | None = None) -> pd.DataFrame: - points = validate_points(points) - ds = xr.open_zarr(STORE, consolidated=True, chunks={}) +def windows(points: pd.DataFrame, days: int): + """Yield (points, start, end) reads that cover each point's own window, its date to `days` later. + + Points in one spatial chunk whose windows overlap share a read, so no chunk is fetched for dates nobody asked for.""" + if "date" not in points or days < 0: + raise ValueError("a window per point needs a date column and days of 0 or more") + pts = points.assign(date=pd.to_datetime(points.date).dt.normalize()) + pts = pts[pts.date.notna()].sort_values("date") # points without a date have no window + if pts.empty: + raise ValueError("no point has a date") + span = pd.Timedelta(days=days) + for _, tile in pts.groupby([(pts[c] // TILE).fillna(-999) for c in ("lat", "lon")]): + run = (tile.date > (tile.date + span).cummax().shift()).cumsum() # a new read where windows stop overlapping + for _, g in tile.groupby(run): + yield g.reset_index(drop=True), g.date.min(), g.date.max() + span + + +def clip(parts: list[pd.DataFrame], points: pd.DataFrame, days: int) -> pd.DataFrame: + """Keep each point's own window and count its days, 0 on the point's date.""" + df = pd.concat(parts, ignore_index=True) + day0 = pd.to_datetime(points.set_index("id").date).dt.normalize() + df.insert(2, "day", (pd.to_datetime(df.date) - df.id.map(day0)).dt.days) + return df[df.day.between(0, days)].sort_values(["id", "date"]).reset_index(drop=True) + + +def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: + """One period for all points (--start), or a window per point (--date-col with --days).""" + if bool(a.start) == bool(a.date_col) or (a.date_col and (a.days is None or a.end)): + p.error("give --start [--end] for one period, or --date-col with --days for a window per point") + points = pd.read_csv(a.points, dtype={"id": str}) + if a.date_col and a.date_col != "date": + if a.date_col not in points: + p.error(f"no column {a.date_col} in {a.points}") + points = points.drop(columns="date", errors="ignore").rename(columns={a.date_col: "date"}) + return points + + +def daily(ds: xr.Dataset, points: pd.DataFrame, start, end) -> pd.DataFrame: sub = select_daily(ds[["Tmax", "Tmin"]], points, start, end, "lat", "lon") sub = sub.where(sub <= TMAX_OK) df = sub.to_dataframe().reset_index().rename(columns={"Tmax": "tmax_c", "Tmin": "tmin_c", "lat": "cell_lat", "lon": "cell_lon"}) @@ -58,13 +122,30 @@ def fetch(points: pd.DataFrame, start: str, end: str | None = None) -> pd.DataFr return df[["id", "date", "tmax_c", "tmin_c", "cell_lat", "cell_lon"]].astype({"tmax_c": "float32", "tmin_c": "float32"}) +def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, days: int | None = None) -> pd.DataFrame: + """One period for all points (start, end), or with days a window per point from its `date` column.""" + points = validate_points(points) + if days is None and start is None: + raise ValueError("give start, or days with a date column in points") + reachable(HOSTS) + ds = open_store() + if days is None: + return daily(ds, points, start, end) + return clip([daily(ds, g, s, e) for g, s, e in windows(points, days)], points, days) + + if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("points"), p.add_argument("out"), p.add_argument("--start", required=True), p.add_argument("--end") + p.add_argument("points"), p.add_argument("out"), p.add_argument("--start"), p.add_argument("--end") + p.add_argument("--date-col", help="column with one date per point, such as the planting date") + p.add_argument("--days", type=int, help="days to fetch after each point's date; 100 gives days 0 to 100") a = p.parse_args() - out = fetch(pd.read_csv(a.points, dtype={"id": str}), a.start, a.end) + points = cli_points(p, a) + out = fetch(points, a.start, a.end, a.days) out.to_csv(a.out, index=False) if a.out.endswith(".csv") else out.to_parquet(a.out, index=False, compression="zstd") sea = out[["tmax_c", "tmin_c"]].isna().all(axis=1).groupby(out.id).all() print(f"{out.id.nunique()} points, {out.date.min()} to {out.date.max()}, {len(out):,} rows -> {a.out}") if sea.any(): print(f"all-NaN (missing coordinates, outside coverage or source nodata): {list(sea[sea].index)}") + if nodate := sorted(set(points.id) - set(out.id)): + print(f"no date, so no window fetched: {nodate}") diff --git a/skills/elevation/scripts/fetch.py b/skills/elevation/scripts/fetch.py index 2e42e31..ae3b6c2 100644 --- a/skills/elevation/scripts/fetch.py +++ b/skills/elevation/scripts/fetch.py @@ -10,6 +10,8 @@ import argparse import math +import urllib.error +import urllib.request from collections import defaultdict import numpy as np @@ -22,6 +24,21 @@ "https://copernicus-dem-90m.s3.amazonaws.com/" "Copernicus_DSM_COG_30_{t}_DEM/Copernicus_DSM_COG_30_{t}_DEM.tif" ) +HOSTS = ["copernicus-dem-90m.s3.amazonaws.com"] # what a firewall must allow + + +def reachable(hosts: list[str]) -> None: + """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" + for host in hosts: + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + pass # the host answered + except OSError as exc: + raise ConnectionError( + f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " + "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." + ) from None def tile_url(lat: float, lon: float) -> str: @@ -33,7 +50,10 @@ def tile_url(lat: float, lon: float) -> str: def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -44,6 +64,7 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: def fetch(points: pd.DataFrame) -> pd.DataFrame: points = validate_points(points) + reachable(HOSTS) out = np.full(len(points), np.nan) by_tile = defaultdict(list) for i, (lat, lon) in enumerate(zip(points.lat, points.lon)): diff --git a/skills/soilgrids-soil/scripts/fetch.py b/skills/soilgrids-soil/scripts/fetch.py index 3b5093d..3e04f33 100644 --- a/skills/soilgrids-soil/scripts/fetch.py +++ b/skills/soilgrids-soil/scripts/fetch.py @@ -9,6 +9,8 @@ points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. Values are in conventional units (see SKILL.md).""" import argparse +import urllib.error +import urllib.request from concurrent.futures import ThreadPoolExecutor import numpy as np @@ -29,6 +31,21 @@ DEFAULT_PROPS = "clay,sand,silt,soc,nitrogen,phh2o,cec,bdod,cfvo" DEFAULT_DEPTHS = "0-5cm,5-15cm,15-30cm" GDAL = dict(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", GDAL_HTTP_MAX_RETRY=3, GDAL_HTTP_RETRY_DELAY=2) +HOSTS = ["files.isric.org"] # what a firewall must allow + + +def reachable(hosts: list[str]) -> None: + """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" + for host in hosts: + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + pass # the host answered + except OSError as exc: + raise ConnectionError( + f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " + "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." + ) from None def layer(prop: str, depth: str, stat: str, res: int, lat: np.ndarray, lon: np.ndarray) -> np.ndarray: @@ -45,7 +62,10 @@ def layer(prop: str, depth: str, stat: str, res: int, lat: np.ndarray, lon: np.n def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -60,6 +80,7 @@ def fetch(points: pd.DataFrame, props: list[str], depths: list[str], stat: str = raise ValueError(f"unknown {sorted(bad)}; properties {', '.join(FACTOR)}; depths {', '.join(DEPTHS)}; stats {', '.join(STATS)}") if res not in URL or (res == 1000 and stat != "mean"): raise ValueError("res is 250 or 1000, and 1000 has the mean only") + reachable(HOSTS) # ocs (carbon stock) exists only for 0-30cm jobs = [(p, "0-30cm" if p == "ocs" else d) for p in props for d in depths] jobs = list(dict.fromkeys(jobs)) From 7f0caee715657cc5c2bfb43cf8b70c33ff47141b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:59:07 +0100 Subject: [PATCH 02/10] test: add host check, offline tests and a weekly live smoke test scripts/check.py uses the standard library only, so it runs where uv cannot install packages. Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01LpE9sWSPBk27r6awN4xP7W --- .github/workflows/test.yml | 35 ++++++++++ scripts/check.py | 80 +++++++++++++++++++++++ tests/points.csv | 3 + tests/smoke.py | 50 ++++++++++++++ tests/test_fetch.py | 130 +++++++++++++++++++++++++++++++++++++ 5 files changed, 298 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 scripts/check.py create mode 100644 tests/points.csv create mode 100644 tests/smoke.py create mode 100644 tests/test_fetch.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..dbb3611 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,35 @@ +name: test + +on: + pull_request: + push: + branches: [main] + schedule: + - cron: "23 5 * * 1" # Mondays, to catch a data store that moved or changed + workflow_dispatch: + +permissions: + contents: read + +jobs: + offline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Unit tests on synthetic stores + run: > + uv run --no-project --python 3.12 --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask + --with fsspec --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests + + live: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Hosts the skills need + run: python3 scripts/check.py + - name: Two points from every store + run: python3 tests/smoke.py + env: + CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} diff --git a/scripts/check.py b/scripts/check.py new file mode 100644 index 0000000..00d5573 --- /dev/null +++ b/scripts/check.py @@ -0,0 +1,80 @@ +"""Can this machine reach what the skills need? Standard library only, so it runs where packages cannot be installed. + + python3 scripts/check.py [skill ...] + +Prints one line per host and exits 1 if any is blocked. Run it first in a sandbox, or after a fetch fails to connect.""" + +import shutil +import sys +import urllib.error +import urllib.request +from urllib.parse import urlsplit + +# host lists mirror HOSTS in each skill's scripts/fetch.py +NEEDS = { + "packages": ["pypi.org", "files.pythonhosted.org"], # uv installs the Python packages from here on first run + "chirps-rainfall": ["dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com"], + "chirts-temperature": ["huggingface.co"], + "agera5-climate": ["arco.datastores.ecmwf.int"], + "elevation": ["copernicus-dem-90m.s3.amazonaws.com"], + "soilgrids-soil": ["files.isric.org"], +} +CHIRTS = "https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr/" +CHUNKS = ["Tmax/0.0.0", "Tmax/0/0/0", "Tmax/c/0/0/0"] # the first chunk under the layouts Zarr uses + + +class Trail(urllib.request.HTTPRedirectHandler): + """Remember each host a download is redirected to.""" + + hosts: list[str] = [] + + def redirect_request(self, req, fp, code, msg, headers, newurl): + self.hosts.append(urlsplit(newurl).hostname) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def blocked(host: str) -> str | None: + """Why the host is out of reach, or None. Any HTTP answer, even 403 or 404, means the host is reachable.""" + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + return None + except OSError as exc: + return str(getattr(exc, "reason", exc)) + return None + + +def chirts_download_hosts() -> dict[str, str | None]: + """Hugging Face serves data files from a second host. Follow one chunk to find it and test it.""" + opener = urllib.request.build_opener(Trail) + for chunk in CHUNKS: + Trail.hosts = [] + try: + opener.open(urllib.request.Request(CHIRTS + chunk, headers={"Range": "bytes=0-0"}), timeout=20) + return {h: None for h in Trail.hosts} + except urllib.error.HTTPError: + continue # not this layout + except OSError as exc: + return {h: (str(getattr(exc, "reason", exc)) if h == Trail.hosts[-1] else None) for h in Trail.hosts} + return {} + + +if __name__ == "__main__": + skills = sys.argv[1:] or [s for s in NEEDS if s != "packages"] + if unknown := set(skills) - set(NEEDS): + sys.exit(f"unknown skill {sorted(unknown)}; choose from {', '.join(s for s in NEEDS if s != 'packages')}") + print(f"uv: {shutil.which('uv') or 'not found, see https://docs.astral.sh/uv/getting-started/installation/'}") + if sys.version_info < (3, 12): + print(f"python {sys.version.split()[0]}: older than 3.12, so uv will also download Python from github.com") + failed = [] + for need in ["packages", *skills]: + results = {h: blocked(h) for h in NEEDS[need]} + if need == "chirts-temperature" and not results["huggingface.co"]: + results |= chirts_download_hosts() + for host, why in results.items(): + print(f"{'BLOCKED' if why else 'ok':8}{need:20}{host}{f' ({why})' if why else ''}") + failed += [host] if why else [] + if failed: + print(f"\nThis network blocks {len(failed)} host(s). Allow them, or run the fetch on a machine with open internet:") + sys.exit("\n".join(failed)) + print("\nAll hosts reachable.") diff --git a/tests/points.csv b/tests/points.csv new file mode 100644 index 0000000..d294e5d --- /dev/null +++ b/tests/points.csv @@ -0,0 +1,3 @@ +id,lat,lon,planting_date +tz-selian,-3.36,36.63,2023-11-01 +bj-niaouli,6.74,2.14,2024-04-15 diff --git a/tests/smoke.py b/tests/smoke.py new file mode 100644 index 0000000..7509415 --- /dev/null +++ b/tests/smoke.py @@ -0,0 +1,50 @@ +"""Fetch two points from every live store and check the answers. Needs uv and open internet. + + python3 tests/smoke.py + +Catches a store that moved, changed layout or stopped answering. AgERA5 runs only when CDSAPI_KEY is set.""" + +import csv +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).parents[1] +WINDOW = ["--date-col", "planting_date", "--days", "5"] # 2 points x days 0 to 5 +# skill: (arguments, columns that must hold values, rows expected) +JOBS = { + "chirps-rainfall": (WINDOW, ["precip_mm"], 12), + "chirts-temperature": (WINDOW, ["tmax_c", "tmin_c"], 12), + "agera5-climate": (WINDOW, ["et0", "srad"], 12), + "elevation": ([], ["elevation_m"], 2), + "soilgrids-soil": (["--props", "clay", "--depths", "0-5cm", "--res", "1000"], ["clay_0_5cm"], 2), +} + + +def fetch(skill: str, args: list[str]) -> list[dict]: + out = Path(tempfile.mkdtemp()) / f"{skill}.csv" + script = ROOT / "skills" / skill / "scripts" / "fetch.py" + subprocess.run(["uv", "run", str(script), str(ROOT / "tests" / "points.csv"), str(out), *args], check=True) + return list(csv.DictReader(out.open())) + + +if __name__ == "__main__": + failed = [] + for skill, (args, columns, n) in JOBS.items(): + if skill == "agera5-climate" and not os.environ.get("CDSAPI_KEY"): + print(f"skipped {skill}: no CDSAPI_KEY") + continue + try: + rows = fetch(skill, args) + assert len(rows) == n, f"{len(rows)} rows, expected {n}" + assert all(r[c] != "" for r in rows for c in columns), f"missing values in {columns}" + print(f"ok {skill}: {rows[0]}") + except (AssertionError, subprocess.CalledProcessError) as exc: + failed.append(skill) + print(f"FAILED {skill}: {exc}") + rows = fetch("chirps-rainfall", ["--start", "2024-01-01", "--end", "2024-01-03"]) # the one-period path + if len(rows) != 6: + failed.append("chirps-rainfall one period") + sys.exit(f"failed: {failed}" if failed else None) diff --git a/tests/test_fetch.py b/tests/test_fetch.py new file mode 100644 index 0000000..fd65dfe --- /dev/null +++ b/tests/test_fetch.py @@ -0,0 +1,130 @@ +"""Offline tests on synthetic stores. No network. + + uv run --no-project --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask --with fsspec \ + --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests +""" + +import argparse +import importlib.util +import inspect +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +ROOT = Path(__file__).parents[1] +WEATHER = ["chirps-rainfall", "chirts-temperature", "agera5-climate"] +SKILLS = [*WEATHER, "elevation", "soilgrids-soil"] + + +def load(path: Path): + spec = importlib.util.spec_from_file_location(path.parent.parent.name.replace("-", "_") + "_" + path.stem, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +MODS = {s: load(ROOT / "skills" / s / "scripts" / "fetch.py") for s in SKILLS} +# per skill: latitude and longitude names, store variables, north-to-south grid, value column, output value on day 10 of the year +STORES = { + "chirps-rainfall": ("latitude", "longitude", ["precipitation_surface"], True, "precip_mm", 86400.0), + "chirts-temperature": ("lat", "lon", ["Tmax", "Tmin"], True, "tmax_c", 1.0), + "agera5-climate": ("latitude", "longitude", [v[0] for v in MODS["agera5-climate"].VARS.values()], False, "et0", 1.0), +} +POINTS = pd.DataFrame({ + "id": ["a", "b", "c", "d"], + "lat": [5.1, -3.3, 5.2, np.nan], # d has no coordinates + "lon": [2.2, 36.7, 2.3, 2.0], + "date": ["2023-02-01", "2023-06-15", None, "2023-03-01"], # c has no date +}) + + +def store(lat: str, lon: str, variables: list[str], descending: bool) -> xr.Dataset: + """A year on a 0.5 degree grid over Africa. Every cell holds a tenth of the day of the year, so a window is easy to check.""" + time = pd.date_range("2023-01-01", "2023-12-31") + lats = np.arange(-9.75, 10, 0.5)[:: -1 if descending else 1] + lons = np.arange(-9.75, 45, 0.5) + doy = np.broadcast_to(np.arange(1, 366, dtype="float32")[:, None, None] / 10, (365, len(lats), len(lons))) + return xr.Dataset({v: (("time", lat, lon), doy.copy()) for v in variables}, coords={"time": time, lat: lats, lon: lons}) + + +@pytest.fixture(params=WEATHER) +def weather(request, monkeypatch): + mod, (lat, lon, variables, descending, column, scale) = MODS[request.param], STORES[request.param] + monkeypatch.setattr(mod, "open_store", lambda: store(lat, lon, variables, descending)) + monkeypatch.setattr(mod, "reachable", lambda hosts: None) + return mod, column, scale + + +def test_shared_helpers_are_identical(): + """Each skill is self-contained, so the helpers are copied. This catches a copy that drifts.""" + shared = {"validate_points": SKILLS, "reachable": SKILLS, "select_daily": WEATHER, "windows": WEATHER, "clip": WEATHER, "cli_points": WEATHER} + for name, skills in shared.items(): + assert len({inspect.getsource(getattr(MODS[s], name)) for s in skills}) == 1, name + + +def test_check_knows_every_host(): + check = load(ROOT / "scripts" / "check.py") + assert {s: check.NEEDS[s] for s in SKILLS} == {s: MODS[s].HOSTS for s in SKILLS} + + +def test_window_per_point(weather): + mod, column, scale = weather + out = mod.fetch(POINTS, days=10) + assert list(out.columns[:3]) == ["id", "date", "day"] + assert set(out.id) == {"a", "b", "d"} # c has no date, so no window + assert out.groupby("id").size().eq(11).all() + a = out[out.id == "a"] + assert list(a.day) == list(range(11)) and str(a.date.iloc[0]) == "2023-02-01" + np.testing.assert_allclose(a[column], np.arange(32, 43) / 10 * scale, rtol=1e-5) # 1 February is day 32 + assert a.cell_lat.iloc[0] == pytest.approx(5.25) and a.cell_lon.iloc[0] == pytest.approx(2.25) + assert out[out.id == "b"][column].iloc[0] == pytest.approx(16.6 * scale, rel=1e-5) # 15 June + assert out[out.id == "d"][column].isna().all() # no coordinates: rows stay, values are NaN + + +def test_one_period_for_all_points(weather): + mod, column, scale = weather + out = mod.fetch(POINTS, "2023-03-01", "2023-03-05") + assert "day" not in out and len(out) == 5 * len(POINTS) + assert out[out.id == "a"][column].iloc[0] == pytest.approx(6.0 * scale, rel=1e-5) + + +def test_window_outside_the_store_fails(weather): + mod, _, _ = weather + late = POINTS.assign(date="2023-12-28") + with pytest.raises(ValueError, match="within"): # never shorten a requested period silently + mod.fetch(late, days=10) + with pytest.raises(ValueError, match="give start"): + mod.fetch(POINTS) + + +def test_overlapping_windows_share_a_read(): + mod = MODS["chirts-temperature"] + pts = pd.DataFrame({"id": list("abcd"), "lat": [5.1, 5.2, 5.3, -20.0], "lon": 2.0, "date": ["2023-02-01", "2023-02-05", "2023-08-01", "2023-02-01"]}) + groups = sorted(mod.windows(mod.validate_points(pts), 10), key=lambda g: (g[1], -len(g[0]))) + assert [list(g.id) for g, _, _ in groups] == [["a", "b"], ["d"], ["c"]] # d sits in another spatial chunk + assert groups[0][1:] == (pd.Timestamp("2023-02-01"), pd.Timestamp("2023-02-15")) + + +@pytest.mark.parametrize("skill", SKILLS) +def test_points_are_validated(skill): + validate = MODS[skill].validate_points + with pytest.raises(ValueError, match="missing"): + validate(pd.DataFrame({"id": ["a"], "latitude": [1.0], "longitude": [2.0]})) + with pytest.raises(ValueError, match="unique"): + validate(pd.DataFrame({"id": ["a", "a"], "lat": [1.0, 1.0], "lon": [2.0, 2.0]})) + with pytest.raises(ValueError, match="outside"): + validate(pd.DataFrame({"id": ["a"], "lat": [36.7], "lon": [-183.3]})) + + +def test_cli_takes_one_mode(tmp_path): + mod = MODS["chirps-rainfall"] + POINTS.rename(columns={"date": "planting_date"}).to_csv(tmp_path / "points.csv", index=False) + parser = argparse.ArgumentParser() + args = dict(points=tmp_path / "points.csv", start=None, end=None, date_col="planting_date", days=10) + assert "date" in mod.cli_points(parser, argparse.Namespace(**args)) + for wrong in ({"days": None}, {"start": "2023-01-01"}, {"date_col": None}, {"date_col": "sowing"}): + with pytest.raises(SystemExit): + mod.cli_points(parser, argparse.Namespace(**(args | wrong))) From b3c2fbc21ff5d400b4eb18ab09c1832bf3503a18 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 13:59:07 +0100 Subject: [PATCH 03/10] docs: document network needs, the stop rule and per-point windows Co-Authored-By: Claude Fable 5.2 Claude-Session: https://claude.ai/code/session_01LpE9sWSPBk27r6awN4xP7W --- AGENTS.md | 2 +- README.md | 18 +++++++++++++- SKILL.md | 39 +++++++++++++++++++++++++++--- llms.txt | 1 + skills/agera5-climate/SKILL.md | 5 +++- skills/chirps-rainfall/SKILL.md | 5 +++- skills/chirts-temperature/SKILL.md | 5 +++- skills/elevation/SKILL.md | 2 ++ skills/soilgrids-soil/SKILL.md | 2 ++ 9 files changed, 70 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 04cf10d..5648c8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This repository holds skills that fetch weather, elevation and soil data for poi 1. Read `SKILL.md` first. It says which skill answers which need and the rules every skill shares. 2. Open only the skill folder the question needs, under `skills/`. Each `SKILL.md` is self-contained and its `scripts/` file does the fetch. -3. Run scripts with `uv run`. Do not install anything else without the user's agreement. +3. Run scripts with `uv run`. Do not install anything else without the user's agreement. If a script cannot reach its data store, or `uv` cannot install packages, follow "Network" in `SKILL.md`: run `python3 scripts/check.py`, name the blocked hosts, and hand the user the exact command. Never estimate values or switch source. 4. Keep the data's caveats with the data. Say which source, grid and dates a result came from. 5. Never invent a DOI, licence or date. Each skill ends with what to cite. 6. Write maniva and tricot in lowercase, always, including headings, table cells and the start of a sentence. Never write "Maniva" or "Tricot". tricot stands for triadic comparison of technology options. Quote published titles exactly as printed, even where they capitalise tricot or expand it as "triadic comparisons of technologies". diff --git a/README.md b/README.md index 4bed01f..d53039d 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,28 @@ ln -s "$PWD"/environment-data/skills/* ~/.claude/skills/ **Codex and Gemini CLI.** Same, into `~/.agents/skills/` or `~/.gemini/skills/`. -**Without an agent.** Run any script directly, e.g. `uv run skills/chirps-rainfall/scripts/fetch.py points.csv rain.parquet --start 2020-01-01`. +**Without an agent.** Run any script directly, e.g. `uv run skills/chirps-rainfall/scripts/fetch.py points.csv rain.parquet --start 2020-01-01`. When each point has its own date, such as a planting date, fetch a window per point: `--date-col planting_date --days 100`. **R users.** Run the scripts as above, then read the parquet outputs in R with `nanoparquet::read_parquet()`, a small package with no dependencies, or `arrow::read_parquet()` if you already use arrow. --- +## Network + +The scripts read remote stores and `uv` installs their packages on first run. Sandboxed agents, such as a cloud chat session, and some institutional networks block those hosts. `python3 scripts/check.py` lists what is blocked, using the standard library only. The hosts are in [SKILL.md](SKILL.md#network), with what an agent does when they are out of reach. + +## Tests + +```sh +uv run --no-project --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask --with fsspec \ + --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests # offline, synthetic stores +python3 tests/smoke.py # two points from every live store +``` + +Both run on every pull request, and the live test runs every Monday to catch a store that moved. + +--- + ## Cite Cite with [CITATION.cff](CITATION.cff). diff --git a/SKILL.md b/SKILL.md index ea7b7b0..ab56a64 100644 --- a/SKILL.md +++ b/SKILL.md @@ -26,22 +26,53 @@ Write maniva and tricot in lowercase, always. That holds in headings, table cell ## Three ways to use a skill -1. **Run the script.** Resolve `scripts/fetch.py` from the selected skill directory and follow its CLI example (weather also requires `--start`). Use this by default. Output is parquet; name the output `.csv` for CSV, which suits elevation, soil and small weather pulls (daily weather for many points gets large). -2. **Import the script module.** Its `fetch()` takes a pandas data frame with `id`, `lat`, `lon` and returns a data frame. Use it inside a larger Python pipeline. +1. **Run the script.** Resolve `scripts/fetch.py` from the selected skill directory and follow its CLI example (weather also requires a period, see "Dates"). Use this by default. Output is parquet; name the output `.csv` for CSV, which suits elevation, soil and small weather pulls (daily weather for many points gets large). +2. **Import the script module.** Its `fetch()` takes a pandas data frame with `id`, `lat`, `lon` (and `date` for a window per point) and returns a data frame. Use it inside a larger Python pipeline. 3. **Write your own reader.** Each skill's "If you need to read ... yourself" section gives the store path and the traps (units, scaling, projection, grid order). Use it for R, other tools or needs the script does not cover. ## Points Users give points in any form: a table with other column names, Excel, tricot data, or coordinates typed in chat. Build the `id`, `lat`, `lon` table yourself, using an existing identifier as `id` or the row number if there is none. Tell the user which columns you used. Use WGS84 decimal degrees with `lat` within −90 to 90 and `lon` within −180 to 180. Preserve IDs as strings and require unique, non-empty IDs for unambiguous joins. Investigate suspected swapped coordinates; never guess a correction or shift points to obtain values. Convert known missing-coordinate sentinels to blanks. Leave rows with missing coordinates in; the scripts return them as NaN. -For a tricot-data trial file, take `site_data[]`: `site_id` → `id`, `latitude` → `lat`, `longitude` → `lon`. Some sites have null coordinates; keep them as blanks. tricot site coordinates are displaced up to 1.5 km for privacy (`metadata.location_privacy`), and some are filled from the mean of their district (`coordinate_source`, `coordinate_uncertainty_m`). That is fine for 5 to 10 km weather grids. For elevation (90 m) and soil (250 m), values describe the published point, not the farm; say so when you report them. +For a tricot-data trial file, take `site_data[]`: `site_id` → `id`, `latitude` → `lat`, `longitude` → `lon`, and keep `planting_date` for a window per site (`--date-col planting_date`). Some sites have null coordinates or planting dates; keep them as blanks. Other trial tables follow the same pattern: one row per plot or block, its identifier as `id`, its coordinates and its planting date. tricot site coordinates are displaced up to 1.5 km for privacy (`metadata.location_privacy`), and some are filled from the mean of their district (`coordinate_source`, `coordinate_uncertainty_m`). That is fine for 5 to 10 km weather grids. For elevation (90 m) and soil (250 m), values describe the published point, not the farm; say so when you report them. + +## Dates + +The weather skills take a period in one of two ways. Use the same one for every source you combine. + +- **One period for all points.** `--start 2020-01-01 --end 2020-12-31`. Use it for a season, a year or a climatology. +- **A window per point.** `--date-col planting_date --days 100` fetches each point from its own date (day 0) to 100 days later, 101 rows per point, with a `day` column counting from 0. Use it whenever points have their own dates, such as planting dates per plot or survey dates per household. Points planted weeks or years apart then cost only their own windows, not the whole span between the first and the last. + +Keep the date column in the points file under its own name and pass that name to `--date-col`. Dates are ISO (`YYYY-MM-DD`). A point with a blank date gets no rows and the script lists it; report those IDs. A window that runs past the store's last day fails rather than being shortened. Ask the user for the number of days if they did not give one; the crop's season length is theirs to state. + +## Network + +The scripts read remote stores, and `uv` installs their packages on first run, so they need these hosts. Sandboxed agents and institutional networks often block them. + +| Need | Hosts | +|---|---| +| Packages, first run of any script | `pypi.org`, `files.pythonhosted.org` | +| `chirps-rainfall` | `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` | +| `chirts-temperature` | `huggingface.co` and the download host it redirects to, which `scripts/check.py` names | +| `agera5-climate` | `arco.datastores.ecmwf.int` | +| `elevation` | `copernicus-dem-90m.s3.amazonaws.com` | +| `soilgrids-soil` | `files.isric.org` | + +`python3 scripts/check.py [skill ...]` tests them with the standard library only, so it runs where nothing can be installed. Each script also checks its store before reading and fails with the blocked host named. + +When a host is blocked: + +1. Run `scripts/check.py` once and report the blocked hosts as it prints them. Do not retry with other tools, mirrors or another data source the user did not ask for. +2. Never estimate, simulate or fill in the values. A blocked network is not missing data. +3. Prepare everything that needs no network: the points file and the exact command. +4. Give the user two ways forward: allow the hosts in the agent's network settings (the change may only apply to a new session), or run the command on their own machine and share the output file. ## Rules 1. Open the one skill the question needs. Each is self-contained. Pass the same points and dates between skills. 2. To run a script, use `uv run`. It needs only `uv`, which fetches Python and the packages on first run. If `uv` is missing, give the user the one-line install from https://docs.astral.sh/uv/getting-started/installation/ and ask before installing. 3. Return the output path, source/store, grid, units, actual dates (or soil depths/statistic), retrieval date and relevant caveats. Values are gridded estimates for a cell, not station readings for a point. -4. Report points and variables with missing values, including partial date gaps. Missing coordinates, coverage limits and source masks are different from network or authentication failures; do not label every failure as water or fill gaps silently. +4. Report points and variables with missing values, including partial date gaps. Missing coordinates, coverage limits and source masks are different from network or authentication failures; do not label every failure as water or fill gaps silently. For a network failure follow "Network". 5. Return daily data when a calling skill asked for it. Start hazard analysis only when the user asked for it. 6. Never invent a DOI, licence or date. Each skill ends with what to cite. diff --git a/llms.txt b/llms.txt index e209777..cb27fca 100644 --- a/llms.txt +++ b/llms.txt @@ -11,6 +11,7 @@ Read SKILL.md first. It says which skill answers which need and the rules they s - [SKILL.md](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/SKILL.md): index of skills and shared rules - [AGENTS.md](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/AGENTS.md): short instructions for AI agents - [README.md](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/README.md): sources, install and licence +- [scripts/check.py](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/scripts/check.py): tests whether this network reaches the data stores; standard library only ## Skills diff --git a/skills/agera5-climate/SKILL.md b/skills/agera5-climate/SKILL.md index 62f4877..5a1f6b5 100644 --- a/skills/agera5-climate/SKILL.md +++ b/skills/agera5-climate/SKILL.md @@ -15,9 +15,10 @@ Resolve `scripts/fetch.py` relative to this skill's directory, not the user's wo ```sh uv run scripts/fetch.py points.csv weather.parquet --start 2020-01-01 --end 2025-12-31 +uv run scripts/fetch.py points.csv weather.parquet --date-col planting_date --days 100 ``` -`--start` is required; use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)) and a key (below). Output is one row per point and day: `id`, `date`, the variables, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs without moving the points. +Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)) and a key (below). Output is one row per point and day: `id`, `date`, the variables, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs without moving the points. Default variables are the ones CHIRPS and CHIRTS don't have. Pick others with `--vars`: @@ -46,6 +47,8 @@ The script looks for `CDSAPI_KEY` in the environment, then the `key:` line of `~ key: ``` +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `arco.datastores.ecmwf.int` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. + ## If you need to read the store yourself ```python diff --git a/skills/chirps-rainfall/SKILL.md b/skills/chirps-rainfall/SKILL.md index c3de7c6..74c1164 100644 --- a/skills/chirps-rainfall/SKILL.md +++ b/skills/chirps-rainfall/SKILL.md @@ -15,9 +15,12 @@ Resolve `scripts/fetch.py` relative to this skill's directory, not the user's wo ```sh uv run scripts/fetch.py points.csv rain.parquet --start 2020-01-01 --end 2025-12-31 +uv run scripts/fetch.py points.csv rain.parquet --date-col planting_date --days 100 ``` -`--start` is required; use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `precip_mm`, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. +Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `precip_mm`, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. + +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. ## If you need to read the store yourself diff --git a/skills/chirts-temperature/SKILL.md b/skills/chirts-temperature/SKILL.md index fc7e7d3..9703a97 100644 --- a/skills/chirts-temperature/SKILL.md +++ b/skills/chirts-temperature/SKILL.md @@ -15,9 +15,12 @@ Resolve `scripts/fetch.py` relative to this skill's directory, not the user's wo ```sh uv run scripts/fetch.py points.csv temp.parquet --start 2020-01-01 --end 2025-12-31 +uv run scripts/fetch.py points.csv temp.parquet --date-col planting_date --days 100 ``` -`--start` is required; use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `tmax_c`, `tmin_c`, `cell_lat`, `cell_lon`. Values above 60 °C are masked (a few source pixels hold impossible values). The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. +Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `tmax_c`, `tmin_c`, `cell_lat`, `cell_lon`. Values above 60 °C are masked (a few source pixels hold impossible values). The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. + +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `huggingface.co` and the download host it redirects to (`scripts/check.py` in the repository names it) for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. ## Bandwidth diff --git a/skills/elevation/SKILL.md b/skills/elevation/SKILL.md index 7a73385..d301d04 100644 --- a/skills/elevation/SKILL.md +++ b/skills/elevation/SKILL.md @@ -19,6 +19,8 @@ uv run scripts/fetch.py points.csv elevation.parquet Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point: `id`, `elevation_m`. The script prints IDs left NaN because coordinates are missing, a tile is absent, or the source cell is nodata. Network and other read errors fail the run. `(0, 0)` is a valid coordinate, not an automatic missing-value sentinel. +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `copernicus-dem-90m.s3.amazonaws.com` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. + ## If you need to read the tiles yourself Tiles are named by their south-west corner, so (-1.95, 30.06) is in `S02_00_E030_00`: diff --git a/skills/soilgrids-soil/SKILL.md b/skills/soilgrids-soil/SKILL.md index f4c1b2d..1d400a4 100644 --- a/skills/soilgrids-soil/SKILL.md +++ b/skills/soilgrids-soil/SKILL.md @@ -25,6 +25,8 @@ Expect about 20 to 30 s for 1,000 points and the default 27 layers. Each layer c Use `--res 1000` when point locations are uncertain by more than a few hundred metres, such as tricot sites, which are displaced up to 1.5 km for privacy. A 1 km average then describes the area the farm is in better than one 250 m cell. It has the mean only, so prediction intervals stay at 250 m. +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `files.isric.org` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. + ## Properties and units Values are converted from SoilGrids' stored integers to the units below. From f2111b112ea90dd972b6f0a95e4dc4d3dd036435 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Tue, 17 Nov 2026 23:55:38 +0100 Subject: [PATCH 04/10] refactor(skills): read all point windows in one vectorised request The tile-and-overlap grouping split reads at window gaps, not chunk edges, so groups in one tile fetched the same chunks again, one compute after another. A (point, day) time index in a single isel lets dask fetch each chunk once, in parallel, and drops TILE, windows() and clip(). On a synthetic 3,640-point store: 232 chunk reads instead of 759. When every point shares its dates, a plain time slice does the read. A per-point index costs dask work for every value, which made 4 years for 3,640 points take 22.7 s instead of 1.7 s. --- skills/agera5-climate/SKILL.md | 2 +- skills/agera5-climate/scripts/fetch.py | 91 +++++++++++----------- skills/chirps-rainfall/SKILL.md | 2 +- skills/chirps-rainfall/scripts/fetch.py | 91 +++++++++++----------- skills/chirts-temperature/SKILL.md | 2 +- skills/chirts-temperature/scripts/fetch.py | 91 +++++++++++----------- tests/test_fetch.py | 10 +-- 7 files changed, 139 insertions(+), 150 deletions(-) diff --git a/skills/agera5-climate/SKILL.md b/skills/agera5-climate/SKILL.md index 5a1f6b5..a5c8ccf 100644 --- a/skills/agera5-climate/SKILL.md +++ b/skills/agera5-climate/SKILL.md @@ -60,7 +60,7 @@ ds = xr.open_zarr("https://arco.datastores.ecmwf.int/cadl-arco-geo-001/arco/sis_ - 27 variables, `ds.data_vars` lists them. Temperatures in K, radiation in J m⁻² d⁻¹, missing is `-9999` and decodes to NaN. - Latitude ascending from -90, longitude from -180. Chunks are 2112 days × 16 × 32 cells, so long periods at a few places are cheap and one day over a region is slow. - Each day runs midnight to midnight local time, not UTC. -- Select variables and dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. +- Select variables and dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. ## Caveats diff --git a/skills/agera5-climate/scripts/fetch.py b/skills/agera5-climate/scripts/fetch.py index 622fa75..0fbb4c4 100644 --- a/skills/agera5-climate/scripts/fetch.py +++ b/skills/agera5-climate/scripts/fetch.py @@ -18,13 +18,13 @@ from pathlib import Path import dask +import numpy as np import pandas as pd import xarray as xr STORE = "https://arco.datastores.ecmwf.int/cadl-arco-geo-001/arco/sis_agrometeorological_indicators/all/geoChunked.zarr" HOSTS = ["arco.datastores.ecmwf.int"] # what a firewall must allow THREADS = 32 # reads wait on the network, not the CPU -TILE = 3 # degrees; close to the store's spatial chunk, so grouped reads rarely fetch a chunk twice # friendly name -> (store variable, scale, offset) so that value = store * scale + offset VARS = { "srad": ("Solar_Radiation_Flux", 1e-6, 0), # J m-2 d-1 -> MJ m-2 d-1 @@ -87,47 +87,49 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: return points -def select_daily(ds, points, start, end, lat, lon): - """Nearest cell per point. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" - first, last = pd.to_datetime(ds.time.values[[0, -1]]) - start, end = pd.Timestamp(start), pd.Timestamp(end or last) - if not first <= start <= end <= last: # else the store silently returns a shorter series - raise ValueError(f"dates must be ordered and within {first.date()} to {last.date()}") +def periods(points: pd.DataFrame, start, end, days) -> tuple[pd.DataFrame, int | None]: + """Each point's day 0 in `date`, and the days after it: one period for all (start, end), or each point's own date and days.""" + if days is None: + if start is None: + raise ValueError("give start, or days with a date column in points") + days = (pd.Timestamp(end) - pd.Timestamp(start)).days if end else None # None: to the store's last day + return points.assign(date=start), days + if "date" not in points: + raise ValueError("a window per point needs a date column") + points = points[pd.to_datetime(points.date).notna()].reset_index(drop=True) # no date, no window + if points.empty: + raise ValueError("no point has a date") + return points, days + + +def select_daily(ds, points, days, lat, lon): + """Nearest cell per point, from its `date` (day 0) to `days` later, or to the store's last day if days is None. + + One vectorised read, so dask fetches each chunk once and in parallel. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" + times = ds.indexes["time"].normalize() + day0 = pd.to_datetime(points.date).dt.normalize().values + days = (times[-1] - day0.min()).days if days is None else days + want = day0[:, None] + np.arange(max(days, 0) + 1) * np.timedelta64(1, "D") + ti = times.get_indexer(want.ravel()).reshape(want.shape) + if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series + raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " + f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") on_grid = points.lat.notna() & points.lon.notna() for col, name in (("lat", lat), ("lon", lon)): axis = ds[name].values half = abs(axis[1] - axis[0]) / 2 on_grid &= points[col].between(axis.min() - half, axis.max() + half) - pts = points[on_grid] - sel = {name: xr.DataArray(pts[col].values, dims="pt", coords={"pt": pts.index}) for col, name in (("lat", lat), ("lon", lon))} + idx = {} + for col, name in (("lat", lat), ("lon", lon)): # off-grid points read any cell, then are masked + idx[name] = xr.DataArray(ds.indexes[name].get_indexer(points[col].where(on_grid, ds[name].values[0]), method="nearest"), dims="pt") + one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index + idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): - sub = ds.sel(time=slice(start, end)).sel(**sel, method="nearest").compute() - return sub.reindex(pt=points.index) - - -def windows(points: pd.DataFrame, days: int): - """Yield (points, start, end) reads that cover each point's own window, its date to `days` later. - - Points in one spatial chunk whose windows overlap share a read, so no chunk is fetched for dates nobody asked for.""" - if "date" not in points or days < 0: - raise ValueError("a window per point needs a date column and days of 0 or more") - pts = points.assign(date=pd.to_datetime(points.date).dt.normalize()) - pts = pts[pts.date.notna()].sort_values("date") # points without a date have no window - if pts.empty: - raise ValueError("no point has a date") - span = pd.Timedelta(days=days) - for _, tile in pts.groupby([(pts[c] // TILE).fillna(-999) for c in ("lat", "lon")]): - run = (tile.date > (tile.date + span).cummax().shift()).cumsum() # a new read where windows stop overlapping - for _, g in tile.groupby(run): - yield g.reset_index(drop=True), g.date.min(), g.date.max() + span - - -def clip(parts: list[pd.DataFrame], points: pd.DataFrame, days: int) -> pd.DataFrame: - """Keep each point's own window and count its days, 0 on the point's date.""" - df = pd.concat(parts, ignore_index=True) - day0 = pd.to_datetime(points.set_index("id").date).dt.normalize() - df.insert(2, "day", (pd.to_datetime(df.date) - df.id.map(day0)).dt.days) - return df[df.day.between(0, days)].sort_values(["id", "date"]).reset_index(drop=True) + sub = ds.isel(idx).compute() + if one: + sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") + keep = xr.DataArray(on_grid.values, dims="pt") + return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: @@ -142,30 +144,27 @@ def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFram return points -def daily(ds: xr.Dataset, points: pd.DataFrame, start, end, names: list[str]) -> pd.DataFrame: - sub = select_daily(ds[[VARS[n][0] for n in names]], points, start, end, "latitude", "longitude") +def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None, names: list[str]) -> pd.DataFrame: + sub = select_daily(ds[[VARS[n][0] for n in names]], points, days, "latitude", "longitude") df = sub.to_dataframe().reset_index().rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) for n in names: var, scale, offset = VARS[n] df[n] = (df.pop(var) * scale + offset).astype("float32") df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date - return df[["id", "date", *names, "cell_lat", "cell_lon"]] + return df[["id", "date", "day", *names, "cell_lat", "cell_lon"]] def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, names: list[str] | None = None, days: int | None = None) -> pd.DataFrame: """One period for all points (start, end), or with days a window per point from its `date` column.""" - points = validate_points(points) names = DEFAULT.split(",") if names is None else names if bad := set(names) - set(VARS): raise ValueError(f"unknown variables {sorted(bad)}; choose from {', '.join(VARS)}") - if days is None and start is None: - raise ValueError("give start, or days with a date column in points") + keep_day = days is not None + points, days = periods(validate_points(points), start, end, days) reachable(HOSTS) - ds = open_store() - if days is None: - return daily(ds, points, start, end, names) - return clip([daily(ds, g, s, e, names) for g, s, e in windows(points, days)], points, days) + out = daily(open_store(), points, days, names) + return out if keep_day else out.drop(columns="day") if __name__ == "__main__": diff --git a/skills/chirps-rainfall/SKILL.md b/skills/chirps-rainfall/SKILL.md index 74c1164..9ce6b15 100644 --- a/skills/chirps-rainfall/SKILL.md +++ b/skills/chirps-rainfall/SKILL.md @@ -34,7 +34,7 @@ ds = xr.open_zarr(icechunk.Repository.open(storage).readonly_session("main").sto - One variable, `precipitation_surface`, in **kg m⁻² s⁻¹**. Multiply by 86400 for mm per day. Dims `(time, latitude, longitude)`. - Latitude is descending. Slice north to south: `latitude=slice(north, south)`. - Land only, 60°S to 60°N. Sea cells are NaN, so check coastal points after fetching. -- Chunks are 365 days × 3° × 3°, so a point read costs one small request per year per 3° tile. Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, and raise dask's thread count (reads are network-bound). Cost scales with the number of 3° tiles the points cover, not the number of points. +- Chunks are 365 days × 3° × 3°, so a point read costs one small request per year per 3° tile. Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, and raise dask's thread count (reads are network-bound). Cost scales with the number of 3° tiles the points cover, not the number of points. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. For a regional map rather than points, the Hugging Face copy `aaguilar90/chirps-v3-daily-rnl` is cheaper (15° chunks); from R it reads with `terra` and GDAL 3.11+, values × 0.1. diff --git a/skills/chirps-rainfall/scripts/fetch.py b/skills/chirps-rainfall/scripts/fetch.py index 4a24228..42023da 100644 --- a/skills/chirps-rainfall/scripts/fetch.py +++ b/skills/chirps-rainfall/scripts/fetch.py @@ -15,6 +15,7 @@ import urllib.request import dask +import numpy as np import icechunk import pandas as pd import xarray as xr @@ -22,7 +23,6 @@ STORE = ("dynamical-ucsb-chc-chirps", "ucsb-chc-chirps-analysis-final/v0.1.0.icechunk") HOSTS = ["dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com"] # what a firewall must allow THREADS = 32 # reads wait on the network, not the CPU -TILE = 3 # degrees; the store's spatial chunk, so grouped reads never fetch a chunk twice def reachable(hosts: list[str]) -> None: @@ -59,47 +59,49 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: return points -def select_daily(ds, points, start, end, lat, lon): - """Nearest cell per point. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" - first, last = pd.to_datetime(ds.time.values[[0, -1]]) - start, end = pd.Timestamp(start), pd.Timestamp(end or last) - if not first <= start <= end <= last: # else the store silently returns a shorter series - raise ValueError(f"dates must be ordered and within {first.date()} to {last.date()}") +def periods(points: pd.DataFrame, start, end, days) -> tuple[pd.DataFrame, int | None]: + """Each point's day 0 in `date`, and the days after it: one period for all (start, end), or each point's own date and days.""" + if days is None: + if start is None: + raise ValueError("give start, or days with a date column in points") + days = (pd.Timestamp(end) - pd.Timestamp(start)).days if end else None # None: to the store's last day + return points.assign(date=start), days + if "date" not in points: + raise ValueError("a window per point needs a date column") + points = points[pd.to_datetime(points.date).notna()].reset_index(drop=True) # no date, no window + if points.empty: + raise ValueError("no point has a date") + return points, days + + +def select_daily(ds, points, days, lat, lon): + """Nearest cell per point, from its `date` (day 0) to `days` later, or to the store's last day if days is None. + + One vectorised read, so dask fetches each chunk once and in parallel. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" + times = ds.indexes["time"].normalize() + day0 = pd.to_datetime(points.date).dt.normalize().values + days = (times[-1] - day0.min()).days if days is None else days + want = day0[:, None] + np.arange(max(days, 0) + 1) * np.timedelta64(1, "D") + ti = times.get_indexer(want.ravel()).reshape(want.shape) + if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series + raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " + f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") on_grid = points.lat.notna() & points.lon.notna() for col, name in (("lat", lat), ("lon", lon)): axis = ds[name].values half = abs(axis[1] - axis[0]) / 2 on_grid &= points[col].between(axis.min() - half, axis.max() + half) - pts = points[on_grid] - sel = {name: xr.DataArray(pts[col].values, dims="pt", coords={"pt": pts.index}) for col, name in (("lat", lat), ("lon", lon))} + idx = {} + for col, name in (("lat", lat), ("lon", lon)): # off-grid points read any cell, then are masked + idx[name] = xr.DataArray(ds.indexes[name].get_indexer(points[col].where(on_grid, ds[name].values[0]), method="nearest"), dims="pt") + one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index + idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): - sub = ds.sel(time=slice(start, end)).sel(**sel, method="nearest").compute() - return sub.reindex(pt=points.index) - - -def windows(points: pd.DataFrame, days: int): - """Yield (points, start, end) reads that cover each point's own window, its date to `days` later. - - Points in one spatial chunk whose windows overlap share a read, so no chunk is fetched for dates nobody asked for.""" - if "date" not in points or days < 0: - raise ValueError("a window per point needs a date column and days of 0 or more") - pts = points.assign(date=pd.to_datetime(points.date).dt.normalize()) - pts = pts[pts.date.notna()].sort_values("date") # points without a date have no window - if pts.empty: - raise ValueError("no point has a date") - span = pd.Timedelta(days=days) - for _, tile in pts.groupby([(pts[c] // TILE).fillna(-999) for c in ("lat", "lon")]): - run = (tile.date > (tile.date + span).cummax().shift()).cumsum() # a new read where windows stop overlapping - for _, g in tile.groupby(run): - yield g.reset_index(drop=True), g.date.min(), g.date.max() + span - - -def clip(parts: list[pd.DataFrame], points: pd.DataFrame, days: int) -> pd.DataFrame: - """Keep each point's own window and count its days, 0 on the point's date.""" - df = pd.concat(parts, ignore_index=True) - day0 = pd.to_datetime(points.set_index("id").date).dt.normalize() - df.insert(2, "day", (pd.to_datetime(df.date) - df.id.map(day0)).dt.days) - return df[df.day.between(0, days)].sort_values(["id", "date"]).reset_index(drop=True) + sub = ds.isel(idx).compute() + if one: + sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") + keep = xr.DataArray(on_grid.values, dims="pt") + return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: @@ -114,25 +116,22 @@ def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFram return points -def daily(ds: xr.Dataset, points: pd.DataFrame, start, end) -> pd.DataFrame: - sub = select_daily(ds.precipitation_surface, points, start, end, "latitude", "longitude") +def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None) -> pd.DataFrame: + sub = select_daily(ds.precipitation_surface, points, days, "latitude", "longitude") df = (sub * 86400).to_dataframe(name="precip_mm").reset_index() # kg m-2 s-1 -> mm/day df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date df = df.rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) - return df[["id", "date", "precip_mm", "cell_lat", "cell_lon"]].astype({"precip_mm": "float32"}) + return df[["id", "date", "day", "precip_mm", "cell_lat", "cell_lon"]].astype({"precip_mm": "float32"}) def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, days: int | None = None) -> pd.DataFrame: """One period for all points (start, end), or with days a window per point from its `date` column.""" - points = validate_points(points) - if days is None and start is None: - raise ValueError("give start, or days with a date column in points") + keep_day = days is not None + points, days = periods(validate_points(points), start, end, days) reachable(HOSTS) - ds = open_store() - if days is None: - return daily(ds, points, start, end) - return clip([daily(ds, g, s, e) for g, s, e in windows(points, days)], points, days) + out = daily(open_store(), points, days) + return out if keep_day else out.drop(columns="day") if __name__ == "__main__": diff --git a/skills/chirts-temperature/SKILL.md b/skills/chirts-temperature/SKILL.md index 9703a97..15510de 100644 --- a/skills/chirts-temperature/SKILL.md +++ b/skills/chirts-temperature/SKILL.md @@ -37,7 +37,7 @@ ds = xr.open_zarr("https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/ - Two variables, `Tmax` and `Tmin`, in °C, stored as int16 in 0.1 °C steps; xarray decodes them and turns `-32768` into NaN. Dims `(time, lat, lon)`. - Latitude is descending. Slice north to south: `lat=slice(north, south)`. - Land only, 60°S to 70°N. Sea and lake cells are NaN every day. -- Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. +- Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. - From R, `terra::rast('ZARR:"path.zarr":/Tmax')` works with GDAL 3.11 or newer, but returns raw integers: multiply by 0.1 and set `NAflag` to -32768. ## Caveats diff --git a/skills/chirts-temperature/scripts/fetch.py b/skills/chirts-temperature/scripts/fetch.py index 5816246..b13cce2 100644 --- a/skills/chirts-temperature/scripts/fetch.py +++ b/skills/chirts-temperature/scripts/fetch.py @@ -15,13 +15,13 @@ import urllib.request import dask +import numpy as np import pandas as pd import xarray as xr STORE = "https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr" HOSTS = ["huggingface.co"] # what a firewall must allow, plus the download host it redirects to (scripts/check.py names it) THREADS = 32 # reads wait on the network, not the CPU -TILE = 15 # degrees; the store's spatial chunk, so grouped reads never fetch a chunk twice TMAX_OK = 60 # a few source pixels hold impossible values; mask above this @@ -58,47 +58,49 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: return points -def select_daily(ds, points, start, end, lat, lon): - """Nearest cell per point. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" - first, last = pd.to_datetime(ds.time.values[[0, -1]]) - start, end = pd.Timestamp(start), pd.Timestamp(end or last) - if not first <= start <= end <= last: # else the store silently returns a shorter series - raise ValueError(f"dates must be ordered and within {first.date()} to {last.date()}") +def periods(points: pd.DataFrame, start, end, days) -> tuple[pd.DataFrame, int | None]: + """Each point's day 0 in `date`, and the days after it: one period for all (start, end), or each point's own date and days.""" + if days is None: + if start is None: + raise ValueError("give start, or days with a date column in points") + days = (pd.Timestamp(end) - pd.Timestamp(start)).days if end else None # None: to the store's last day + return points.assign(date=start), days + if "date" not in points: + raise ValueError("a window per point needs a date column") + points = points[pd.to_datetime(points.date).notna()].reset_index(drop=True) # no date, no window + if points.empty: + raise ValueError("no point has a date") + return points, days + + +def select_daily(ds, points, days, lat, lon): + """Nearest cell per point, from its `date` (day 0) to `days` later, or to the store's last day if days is None. + + One vectorised read, so dask fetches each chunk once and in parallel. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" + times = ds.indexes["time"].normalize() + day0 = pd.to_datetime(points.date).dt.normalize().values + days = (times[-1] - day0.min()).days if days is None else days + want = day0[:, None] + np.arange(max(days, 0) + 1) * np.timedelta64(1, "D") + ti = times.get_indexer(want.ravel()).reshape(want.shape) + if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series + raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " + f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") on_grid = points.lat.notna() & points.lon.notna() for col, name in (("lat", lat), ("lon", lon)): axis = ds[name].values half = abs(axis[1] - axis[0]) / 2 on_grid &= points[col].between(axis.min() - half, axis.max() + half) - pts = points[on_grid] - sel = {name: xr.DataArray(pts[col].values, dims="pt", coords={"pt": pts.index}) for col, name in (("lat", lat), ("lon", lon))} + idx = {} + for col, name in (("lat", lat), ("lon", lon)): # off-grid points read any cell, then are masked + idx[name] = xr.DataArray(ds.indexes[name].get_indexer(points[col].where(on_grid, ds[name].values[0]), method="nearest"), dims="pt") + one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index + idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): - sub = ds.sel(time=slice(start, end)).sel(**sel, method="nearest").compute() - return sub.reindex(pt=points.index) - - -def windows(points: pd.DataFrame, days: int): - """Yield (points, start, end) reads that cover each point's own window, its date to `days` later. - - Points in one spatial chunk whose windows overlap share a read, so no chunk is fetched for dates nobody asked for.""" - if "date" not in points or days < 0: - raise ValueError("a window per point needs a date column and days of 0 or more") - pts = points.assign(date=pd.to_datetime(points.date).dt.normalize()) - pts = pts[pts.date.notna()].sort_values("date") # points without a date have no window - if pts.empty: - raise ValueError("no point has a date") - span = pd.Timedelta(days=days) - for _, tile in pts.groupby([(pts[c] // TILE).fillna(-999) for c in ("lat", "lon")]): - run = (tile.date > (tile.date + span).cummax().shift()).cumsum() # a new read where windows stop overlapping - for _, g in tile.groupby(run): - yield g.reset_index(drop=True), g.date.min(), g.date.max() + span - - -def clip(parts: list[pd.DataFrame], points: pd.DataFrame, days: int) -> pd.DataFrame: - """Keep each point's own window and count its days, 0 on the point's date.""" - df = pd.concat(parts, ignore_index=True) - day0 = pd.to_datetime(points.set_index("id").date).dt.normalize() - df.insert(2, "day", (pd.to_datetime(df.date) - df.id.map(day0)).dt.days) - return df[df.day.between(0, days)].sort_values(["id", "date"]).reset_index(drop=True) + sub = ds.isel(idx).compute() + if one: + sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") + keep = xr.DataArray(on_grid.values, dims="pt") + return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: @@ -113,25 +115,22 @@ def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFram return points -def daily(ds: xr.Dataset, points: pd.DataFrame, start, end) -> pd.DataFrame: - sub = select_daily(ds[["Tmax", "Tmin"]], points, start, end, "lat", "lon") +def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None) -> pd.DataFrame: + sub = select_daily(ds[["Tmax", "Tmin"]], points, days, "lat", "lon") sub = sub.where(sub <= TMAX_OK) df = sub.to_dataframe().reset_index().rename(columns={"Tmax": "tmax_c", "Tmin": "tmin_c", "lat": "cell_lat", "lon": "cell_lon"}) df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date - return df[["id", "date", "tmax_c", "tmin_c", "cell_lat", "cell_lon"]].astype({"tmax_c": "float32", "tmin_c": "float32"}) + return df[["id", "date", "day", "tmax_c", "tmin_c", "cell_lat", "cell_lon"]].astype({"tmax_c": "float32", "tmin_c": "float32"}) def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, days: int | None = None) -> pd.DataFrame: """One period for all points (start, end), or with days a window per point from its `date` column.""" - points = validate_points(points) - if days is None and start is None: - raise ValueError("give start, or days with a date column in points") + keep_day = days is not None + points, days = periods(validate_points(points), start, end, days) reachable(HOSTS) - ds = open_store() - if days is None: - return daily(ds, points, start, end) - return clip([daily(ds, g, s, e) for g, s, e in windows(points, days)], points, days) + out = daily(open_store(), points, days) + return out if keep_day else out.drop(columns="day") if __name__ == "__main__": diff --git a/tests/test_fetch.py b/tests/test_fetch.py index fd65dfe..75c0140 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -60,7 +60,7 @@ def weather(request, monkeypatch): def test_shared_helpers_are_identical(): """Each skill is self-contained, so the helpers are copied. This catches a copy that drifts.""" - shared = {"validate_points": SKILLS, "reachable": SKILLS, "select_daily": WEATHER, "windows": WEATHER, "clip": WEATHER, "cli_points": WEATHER} + shared = {"validate_points": SKILLS, "reachable": SKILLS, "select_daily": WEATHER, "periods": WEATHER, "cli_points": WEATHER} for name, skills in shared.items(): assert len({inspect.getsource(getattr(MODS[s], name)) for s in skills}) == 1, name @@ -100,14 +100,6 @@ def test_window_outside_the_store_fails(weather): mod.fetch(POINTS) -def test_overlapping_windows_share_a_read(): - mod = MODS["chirts-temperature"] - pts = pd.DataFrame({"id": list("abcd"), "lat": [5.1, 5.2, 5.3, -20.0], "lon": 2.0, "date": ["2023-02-01", "2023-02-05", "2023-08-01", "2023-02-01"]}) - groups = sorted(mod.windows(mod.validate_points(pts), 10), key=lambda g: (g[1], -len(g[0]))) - assert [list(g.id) for g, _, _ in groups] == [["a", "b"], ["d"], ["c"]] # d sits in another spatial chunk - assert groups[0][1:] == (pd.Timestamp("2023-02-01"), pd.Timestamp("2023-02-15")) - - @pytest.mark.parametrize("skill", SKILLS) def test_points_are_validated(skill): validate = MODS[skill].validate_points From c02298afdf30675b19df73815bb8d3e94ee0c69c Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 00:06:27 +0100 Subject: [PATCH 05/10] refactor(skills): find off-grid points with a nearest-match tolerance pandas get_indexer with a half-cell tolerance returns -1 for off-grid and missing coordinates, replacing the hand-written bounds check. The point id rides along as the pt coordinate, so daily() no longer maps it back by position. A 1e-6 slack keeps points on a cell edge on the grid. --- skills/agera5-climate/scripts/fetch.py | 16 ++++++---------- skills/chirps-rainfall/scripts/fetch.py | 16 ++++++---------- skills/chirts-temperature/scripts/fetch.py | 16 ++++++---------- tests/test_fetch.py | 6 ++++++ 4 files changed, 24 insertions(+), 30 deletions(-) diff --git a/skills/agera5-climate/scripts/fetch.py b/skills/agera5-climate/scripts/fetch.py index 0fbb4c4..d6c006b 100644 --- a/skills/agera5-climate/scripts/fetch.py +++ b/skills/agera5-climate/scripts/fetch.py @@ -114,21 +114,18 @@ def select_daily(ds, points, days, lat, lon): if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") - on_grid = points.lat.notna() & points.lon.notna() - for col, name in (("lat", lat), ("lon", lon)): - axis = ds[name].values - half = abs(axis[1] - axis[0]) / 2 - on_grid &= points[col].between(axis.min() - half, axis.max() + half) idx = {} - for col, name in (("lat", lat), ("lon", lon)): # off-grid points read any cell, then are masked - idx[name] = xr.DataArray(ds.indexes[name].get_indexer(points[col].where(on_grid, ds[name].values[0]), method="nearest"), dims="pt") + for col, name in (("lat", lat), ("lon", lon)): # -1 when off grid or missing; it reads the last cell, then is masked + half = abs(float(ds[name][1] - ds[name][0])) / 2 + 1e-6 # slack: points on a cell edge must not round off grid + near = ds.indexes[name].get_indexer(points[col], method="nearest", tolerance=half) + idx[name] = xr.DataArray(near, dims="pt", coords={"pt": points.id.values}) one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): sub = ds.isel(idx).compute() if one: sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") - keep = xr.DataArray(on_grid.values, dims="pt") + keep = (idx[lat] >= 0) & (idx[lon] >= 0) return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) @@ -146,11 +143,10 @@ def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFram def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None, names: list[str]) -> pd.DataFrame: sub = select_daily(ds[[VARS[n][0] for n in names]], points, days, "latitude", "longitude") - df = sub.to_dataframe().reset_index().rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) + df = sub.to_dataframe().reset_index().rename(columns={"pt": "id", "latitude": "cell_lat", "longitude": "cell_lon"}) for n in names: var, scale, offset = VARS[n] df[n] = (df.pop(var) * scale + offset).astype("float32") - df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date return df[["id", "date", "day", *names, "cell_lat", "cell_lon"]] diff --git a/skills/chirps-rainfall/scripts/fetch.py b/skills/chirps-rainfall/scripts/fetch.py index 42023da..0f6cabb 100644 --- a/skills/chirps-rainfall/scripts/fetch.py +++ b/skills/chirps-rainfall/scripts/fetch.py @@ -86,21 +86,18 @@ def select_daily(ds, points, days, lat, lon): if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") - on_grid = points.lat.notna() & points.lon.notna() - for col, name in (("lat", lat), ("lon", lon)): - axis = ds[name].values - half = abs(axis[1] - axis[0]) / 2 - on_grid &= points[col].between(axis.min() - half, axis.max() + half) idx = {} - for col, name in (("lat", lat), ("lon", lon)): # off-grid points read any cell, then are masked - idx[name] = xr.DataArray(ds.indexes[name].get_indexer(points[col].where(on_grid, ds[name].values[0]), method="nearest"), dims="pt") + for col, name in (("lat", lat), ("lon", lon)): # -1 when off grid or missing; it reads the last cell, then is masked + half = abs(float(ds[name][1] - ds[name][0])) / 2 + 1e-6 # slack: points on a cell edge must not round off grid + near = ds.indexes[name].get_indexer(points[col], method="nearest", tolerance=half) + idx[name] = xr.DataArray(near, dims="pt", coords={"pt": points.id.values}) one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): sub = ds.isel(idx).compute() if one: sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") - keep = xr.DataArray(on_grid.values, dims="pt") + keep = (idx[lat] >= 0) & (idx[lon] >= 0) return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) @@ -119,9 +116,8 @@ def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFram def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None) -> pd.DataFrame: sub = select_daily(ds.precipitation_surface, points, days, "latitude", "longitude") df = (sub * 86400).to_dataframe(name="precip_mm").reset_index() # kg m-2 s-1 -> mm/day - df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date - df = df.rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) + df = df.rename(columns={"pt": "id", "latitude": "cell_lat", "longitude": "cell_lon"}) return df[["id", "date", "day", "precip_mm", "cell_lat", "cell_lon"]].astype({"precip_mm": "float32"}) diff --git a/skills/chirts-temperature/scripts/fetch.py b/skills/chirts-temperature/scripts/fetch.py index b13cce2..6c73293 100644 --- a/skills/chirts-temperature/scripts/fetch.py +++ b/skills/chirts-temperature/scripts/fetch.py @@ -85,21 +85,18 @@ def select_daily(ds, points, days, lat, lon): if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") - on_grid = points.lat.notna() & points.lon.notna() - for col, name in (("lat", lat), ("lon", lon)): - axis = ds[name].values - half = abs(axis[1] - axis[0]) / 2 - on_grid &= points[col].between(axis.min() - half, axis.max() + half) idx = {} - for col, name in (("lat", lat), ("lon", lon)): # off-grid points read any cell, then are masked - idx[name] = xr.DataArray(ds.indexes[name].get_indexer(points[col].where(on_grid, ds[name].values[0]), method="nearest"), dims="pt") + for col, name in (("lat", lat), ("lon", lon)): # -1 when off grid or missing; it reads the last cell, then is masked + half = abs(float(ds[name][1] - ds[name][0])) / 2 + 1e-6 # slack: points on a cell edge must not round off grid + near = ds.indexes[name].get_indexer(points[col], method="nearest", tolerance=half) + idx[name] = xr.DataArray(near, dims="pt", coords={"pt": points.id.values}) one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): sub = ds.isel(idx).compute() if one: sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") - keep = xr.DataArray(on_grid.values, dims="pt") + keep = (idx[lat] >= 0) & (idx[lon] >= 0) return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) @@ -118,8 +115,7 @@ def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFram def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None) -> pd.DataFrame: sub = select_daily(ds[["Tmax", "Tmin"]], points, days, "lat", "lon") sub = sub.where(sub <= TMAX_OK) - df = sub.to_dataframe().reset_index().rename(columns={"Tmax": "tmax_c", "Tmin": "tmin_c", "lat": "cell_lat", "lon": "cell_lon"}) - df["id"] = points.id.values[df.pt] + df = sub.to_dataframe().reset_index().rename(columns={"pt": "id", "Tmax": "tmax_c", "Tmin": "tmin_c", "lat": "cell_lat", "lon": "cell_lon"}) df["date"] = df.time.dt.date return df[["id", "date", "day", "tmax_c", "tmin_c", "cell_lat", "cell_lon"]].astype({"tmax_c": "float32", "tmin_c": "float32"}) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 75c0140..78f5c4b 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -91,6 +91,12 @@ def test_one_period_for_all_points(weather): assert out[out.id == "a"][column].iloc[0] == pytest.approx(6.0 * scale, rel=1e-5) +def test_point_on_a_cell_edge_is_on_grid(weather): + mod, column, _ = weather + edge = pd.DataFrame({"id": ["e"], "lat": [5.0 + 1e-9], "lon": [2.0]}) # float noise past the edge between two cells + assert mod.fetch(edge, "2023-03-01", "2023-03-02")[column].notna().all() + + def test_window_outside_the_store_fails(weather): mod, _, _ = weather late = POINTS.assign(date="2023-12-28") From 53e8113117245127c8de65c89d0e67d10f628706 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 09:50:53 +0100 Subject: [PATCH 06/10] refactor(skills): replace the reachability pre-check with a wrapped read A failed read already fails fast and says why, so the extra HEAD request on every run added nothing. Wrapping the read names the store host and points to scripts/check.py, and also catches the CHIRTS download host that the pre-check never tested. --- SKILL.md | 2 +- skills/agera5-climate/scripts/fetch.py | 22 ++++-------------- skills/chirps-rainfall/scripts/fetch.py | 22 ++++-------------- skills/chirts-temperature/scripts/fetch.py | 22 ++++-------------- skills/elevation/scripts/fetch.py | 19 +--------------- skills/soilgrids-soil/scripts/fetch.py | 26 +++++----------------- tests/test_fetch.py | 10 +++++++-- 7 files changed, 28 insertions(+), 95 deletions(-) diff --git a/SKILL.md b/SKILL.md index ab56a64..ce7ded6 100644 --- a/SKILL.md +++ b/SKILL.md @@ -58,7 +58,7 @@ The scripts read remote stores, and `uv` installs their packages on first run, s | `elevation` | `copernicus-dem-90m.s3.amazonaws.com` | | `soilgrids-soil` | `files.isric.org` | -`python3 scripts/check.py [skill ...]` tests them with the standard library only, so it runs where nothing can be installed. Each script also checks its store before reading and fails with the blocked host named. +`python3 scripts/check.py [skill ...]` tests them with the standard library only, so it runs where nothing can be installed. When a script cannot read its store, its error names the host and points here. When a host is blocked: diff --git a/skills/agera5-climate/scripts/fetch.py b/skills/agera5-climate/scripts/fetch.py index d6c006b..be1744f 100644 --- a/skills/agera5-climate/scripts/fetch.py +++ b/skills/agera5-climate/scripts/fetch.py @@ -13,8 +13,6 @@ import argparse import os -import urllib.error -import urllib.request from pathlib import Path import dask @@ -53,20 +51,6 @@ def cds_key() -> str: raise SystemExit("No CDS key. Set CDSAPI_KEY or write ~/.cdsapirc; see SKILL.md for how to get one.") -def reachable(hosts: list[str]) -> None: - """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" - for host in hosts: - try: - urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) - except urllib.error.HTTPError: - pass # the host answered - except OSError as exc: - raise ConnectionError( - f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " - "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." - ) from None - - def open_store() -> xr.Dataset: # trust_env makes the HTTP client honour HTTPS_PROXY, which sandboxes and institutional networks route through options = {"headers": {"Authorization": f"Bearer {cds_key()}"}, "client_kwargs": {"trust_env": True}} @@ -158,8 +142,10 @@ def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None raise ValueError(f"unknown variables {sorted(bad)}; choose from {', '.join(VARS)}") keep_day = days is not None points, days = periods(validate_points(points), start, end, days) - reachable(HOSTS) - out = daily(open_store(), points, days, names) + try: + out = daily(open_store(), points, days, names) + except OSError as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return out if keep_day else out.drop(columns="day") diff --git a/skills/chirps-rainfall/scripts/fetch.py b/skills/chirps-rainfall/scripts/fetch.py index 0f6cabb..321df38 100644 --- a/skills/chirps-rainfall/scripts/fetch.py +++ b/skills/chirps-rainfall/scripts/fetch.py @@ -11,8 +11,6 @@ With --date-col each point gets its own window, from its date (day 0) to --days later.""" import argparse -import urllib.error -import urllib.request import dask import numpy as np @@ -25,20 +23,6 @@ THREADS = 32 # reads wait on the network, not the CPU -def reachable(hosts: list[str]) -> None: - """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" - for host in hosts: - try: - urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) - except urllib.error.HTTPError: - pass # the host answered - except OSError as exc: - raise ConnectionError( - f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " - "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." - ) from None - - def open_store() -> xr.Dataset: storage = icechunk.s3_storage(bucket=STORE[0], prefix=STORE[1], region="us-west-2", anonymous=True) session = icechunk.Repository.open(storage).readonly_session("main") @@ -125,8 +109,10 @@ def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None """One period for all points (start, end), or with days a window per point from its `date` column.""" keep_day = days is not None points, days = periods(validate_points(points), start, end, days) - reachable(HOSTS) - out = daily(open_store(), points, days) + try: + out = daily(open_store(), points, days) + except (OSError, icechunk.StorageError) as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return out if keep_day else out.drop(columns="day") diff --git a/skills/chirts-temperature/scripts/fetch.py b/skills/chirts-temperature/scripts/fetch.py index 6c73293..2e3dc6f 100644 --- a/skills/chirts-temperature/scripts/fetch.py +++ b/skills/chirts-temperature/scripts/fetch.py @@ -11,8 +11,6 @@ With --date-col each point gets its own window, from its date (day 0) to --days later.""" import argparse -import urllib.error -import urllib.request import dask import numpy as np @@ -25,20 +23,6 @@ TMAX_OK = 60 # a few source pixels hold impossible values; mask above this -def reachable(hosts: list[str]) -> None: - """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" - for host in hosts: - try: - urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) - except urllib.error.HTTPError: - pass # the host answered - except OSError as exc: - raise ConnectionError( - f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " - "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." - ) from None - - def open_store() -> xr.Dataset: # trust_env makes the HTTP client honour HTTPS_PROXY, which sandboxes and institutional networks route through return xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"client_kwargs": {"trust_env": True}}) @@ -124,8 +108,10 @@ def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None """One period for all points (start, end), or with days a window per point from its `date` column.""" keep_day = days is not None points, days = periods(validate_points(points), start, end, days) - reachable(HOSTS) - out = daily(open_store(), points, days) + try: + out = daily(open_store(), points, days) + except OSError as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return out if keep_day else out.drop(columns="day") diff --git a/skills/elevation/scripts/fetch.py b/skills/elevation/scripts/fetch.py index ae3b6c2..947d19c 100644 --- a/skills/elevation/scripts/fetch.py +++ b/skills/elevation/scripts/fetch.py @@ -10,8 +10,6 @@ import argparse import math -import urllib.error -import urllib.request from collections import defaultdict import numpy as np @@ -27,20 +25,6 @@ HOSTS = ["copernicus-dem-90m.s3.amazonaws.com"] # what a firewall must allow -def reachable(hosts: list[str]) -> None: - """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" - for host in hosts: - try: - urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) - except urllib.error.HTTPError: - pass # the host answered - except OSError as exc: - raise ConnectionError( - f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " - "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." - ) from None - - def tile_url(lat: float, lon: float) -> str: """Tiles are 1 degree, named by their south-west corner: S02_00_E029_00 covers 2-1 S, 29-30 E.""" y, x = math.floor(lat), math.floor(lon) @@ -64,7 +48,6 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: def fetch(points: pd.DataFrame) -> pd.DataFrame: points = validate_points(points) - reachable(HOSTS) out = np.full(len(points), np.nan) by_tile = defaultdict(list) for i, (lat, lon) in enumerate(zip(points.lat, points.lon)): @@ -79,7 +62,7 @@ def fetch(points: pd.DataFrame) -> pd.DataFrame: except RasterioIOError as exc: # A missing ocean tile is expected; a failed connection is not nodata. if "404" not in str(exc): - raise + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return pd.DataFrame({"id": points.id, "elevation_m": out.astype("float32")}) diff --git a/skills/soilgrids-soil/scripts/fetch.py b/skills/soilgrids-soil/scripts/fetch.py index 3e04f33..c04234b 100644 --- a/skills/soilgrids-soil/scripts/fetch.py +++ b/skills/soilgrids-soil/scripts/fetch.py @@ -9,8 +9,6 @@ points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. Values are in conventional units (see SKILL.md).""" import argparse -import urllib.error -import urllib.request from concurrent.futures import ThreadPoolExecutor import numpy as np @@ -34,20 +32,6 @@ HOSTS = ["files.isric.org"] # what a firewall must allow -def reachable(hosts: list[str]) -> None: - """Fail early and name the host when a firewall or sandbox blocks the store. Any HTTP answer counts as reachable.""" - for host in hosts: - try: - urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) - except urllib.error.HTTPError: - pass # the host answered - except OSError as exc: - raise ConnectionError( - f"cannot reach {host} ({exc}). This is a network block, not missing data. Allow {', '.join(hosts)} " - "or run the same command on a machine with open internet. See 'Network' in the repository SKILL.md." - ) from None - - def layer(prop: str, depth: str, stat: str, res: int, lat: np.ndarray, lon: np.ndarray) -> np.ndarray: v = np.full(len(lat), np.nan, dtype="float32") ok = np.isfinite(lat) & np.isfinite(lon) # blank coordinates stay NaN @@ -80,14 +64,16 @@ def fetch(points: pd.DataFrame, props: list[str], depths: list[str], stat: str = raise ValueError(f"unknown {sorted(bad)}; properties {', '.join(FACTOR)}; depths {', '.join(DEPTHS)}; stats {', '.join(STATS)}") if res not in URL or (res == 1000 and stat != "mean"): raise ValueError("res is 250 or 1000, and 1000 has the mean only") - reachable(HOSTS) # ocs (carbon stock) exists only for 0-30cm jobs = [(p, "0-30cm" if p == "ocs" else d) for p in props for d in depths] jobs = list(dict.fromkeys(jobs)) lat, lon = points.lat.to_numpy(float), points.lon.to_numpy(float) - with ThreadPoolExecutor(min(32, len(jobs))) as ex: # each layer opens its own 6 MB VRT; network-bound - cols = ex.map(lambda j: layer(*j, stat, res, lat, lon), jobs) - out = {f"{p}_{d.replace('-', '_')}": c for (p, d), c in zip(jobs, cols)} + try: + with ThreadPoolExecutor(min(32, len(jobs))) as ex: # each layer opens its own 6 MB VRT; network-bound + cols = ex.map(lambda j: layer(*j, stat, res, lat, lon), jobs) + out = {f"{p}_{d.replace('-', '_')}": c for (p, d), c in zip(jobs, cols)} + except OSError as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return pd.DataFrame({"id": points.id, **out}) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 78f5c4b..1e40f76 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -54,13 +54,12 @@ def store(lat: str, lon: str, variables: list[str], descending: bool) -> xr.Data def weather(request, monkeypatch): mod, (lat, lon, variables, descending, column, scale) = MODS[request.param], STORES[request.param] monkeypatch.setattr(mod, "open_store", lambda: store(lat, lon, variables, descending)) - monkeypatch.setattr(mod, "reachable", lambda hosts: None) return mod, column, scale def test_shared_helpers_are_identical(): """Each skill is self-contained, so the helpers are copied. This catches a copy that drifts.""" - shared = {"validate_points": SKILLS, "reachable": SKILLS, "select_daily": WEATHER, "periods": WEATHER, "cli_points": WEATHER} + shared = {"validate_points": SKILLS, "select_daily": WEATHER, "periods": WEATHER, "cli_points": WEATHER} for name, skills in shared.items(): assert len({inspect.getsource(getattr(MODS[s], name)) for s in skills}) == 1, name @@ -97,6 +96,13 @@ def test_point_on_a_cell_edge_is_on_grid(weather): assert mod.fetch(edge, "2023-03-01", "2023-03-02")[column].notna().all() +def test_a_failed_read_points_to_check(weather, monkeypatch): + mod, _, _ = weather + monkeypatch.setattr(mod, "open_store", lambda: (_ for _ in ()).throw(OSError("Cannot connect to host 127.0.0.1:9"))) + with pytest.raises(ConnectionError, match="check.py"): + mod.fetch(POINTS, "2023-03-01", "2023-03-02") + + def test_window_outside_the_store_fails(weather): mod, _, _ = weather late = POINTS.assign(date="2023-12-28") From 9ee34c059bcb17a8ca7d838d33baf037605baf60 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 11:56:11 +0100 Subject: [PATCH 07/10] ci: run the live smoke test only when skills change, and weekly The host check only tested GitHub's network, which always reaches the stores, and the smoke test reads from them anyway. Docs-only pull requests no longer fail when a remote store is slow. --- .github/workflows/live.yml | 25 +++++++++++++++++++++++++ .github/workflows/test.yml | 15 --------------- README.md | 2 +- 3 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/live.yml diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml new file mode 100644 index 0000000..79ba003 --- /dev/null +++ b/.github/workflows/live.yml @@ -0,0 +1,25 @@ +name: live + +on: + pull_request: + paths: ["skills/**", "tests/**"] + push: + branches: [main] + paths: ["skills/**", "tests/**"] + schedule: + - cron: "23 5 * * 1" # Mondays, to catch a data store that moved or changed + workflow_dispatch: + +permissions: + contents: read + +jobs: + live: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Two points from every store + run: python3 tests/smoke.py + env: + CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dbb3611..7a6bb47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,9 +4,6 @@ on: pull_request: push: branches: [main] - schedule: - - cron: "23 5 * * 1" # Mondays, to catch a data store that moved or changed - workflow_dispatch: permissions: contents: read @@ -21,15 +18,3 @@ jobs: run: > uv run --no-project --python 3.12 --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask --with fsspec --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests - - live: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 - - name: Hosts the skills need - run: python3 scripts/check.py - - name: Two points from every store - run: python3 tests/smoke.py - env: - CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} diff --git a/README.md b/README.md index d53039d..43b0b7f 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ uv run --no-project --with pytest --with xarray --with "zarr>=3" --with icechunk python3 tests/smoke.py # two points from every live store ``` -Both run on every pull request, and the live test runs every Monday to catch a store that moved. +The offline tests run on every pull request. The live test runs on pull requests that change `skills/` or `tests/`, and every Monday to catch a store that moved. --- From fef1a62c5a292b636bfb7299d9d8dbb1e09c3b3b Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 12:12:37 +0100 Subject: [PATCH 08/10] test(smoke): run the one-period check as an ordinary job It ran outside the try, so a failure crashed the script instead of being reported with the rest. --- tests/smoke.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/smoke.py b/tests/smoke.py index 7509415..1bf534b 100644 --- a/tests/smoke.py +++ b/tests/smoke.py @@ -13,14 +13,15 @@ ROOT = Path(__file__).parents[1] WINDOW = ["--date-col", "planting_date", "--days", "5"] # 2 points x days 0 to 5 -# skill: (arguments, columns that must hold values, rows expected) -JOBS = { - "chirps-rainfall": (WINDOW, ["precip_mm"], 12), - "chirts-temperature": (WINDOW, ["tmax_c", "tmin_c"], 12), - "agera5-climate": (WINDOW, ["et0", "srad"], 12), - "elevation": ([], ["elevation_m"], 2), - "soilgrids-soil": (["--props", "clay", "--depths", "0-5cm", "--res", "1000"], ["clay_0_5cm"], 2), -} +# (label, skill, arguments, columns that must hold values, rows expected) +JOBS = [ + ("chirps-rainfall", "chirps-rainfall", WINDOW, ["precip_mm"], 12), + ("chirps-rainfall one period", "chirps-rainfall", ["--start", "2024-01-01", "--end", "2024-01-03"], ["precip_mm"], 6), + ("chirts-temperature", "chirts-temperature", WINDOW, ["tmax_c", "tmin_c"], 12), + ("agera5-climate", "agera5-climate", WINDOW, ["et0", "srad"], 12), + ("elevation", "elevation", [], ["elevation_m"], 2), + ("soilgrids-soil", "soilgrids-soil", ["--props", "clay", "--depths", "0-5cm", "--res", "1000"], ["clay_0_5cm"], 2), +] def fetch(skill: str, args: list[str]) -> list[dict]: @@ -32,19 +33,16 @@ def fetch(skill: str, args: list[str]) -> list[dict]: if __name__ == "__main__": failed = [] - for skill, (args, columns, n) in JOBS.items(): + for label, skill, args, columns, n in JOBS: if skill == "agera5-climate" and not os.environ.get("CDSAPI_KEY"): - print(f"skipped {skill}: no CDSAPI_KEY") + print(f"skipped {label}: no CDSAPI_KEY") continue try: rows = fetch(skill, args) assert len(rows) == n, f"{len(rows)} rows, expected {n}" assert all(r[c] != "" for r in rows for c in columns), f"missing values in {columns}" - print(f"ok {skill}: {rows[0]}") + print(f"ok {label}: {rows[0]}") except (AssertionError, subprocess.CalledProcessError) as exc: - failed.append(skill) - print(f"FAILED {skill}: {exc}") - rows = fetch("chirps-rainfall", ["--start", "2024-01-01", "--end", "2024-01-03"]) # the one-period path - if len(rows) != 6: - failed.append("chirps-rainfall one period") + failed.append(label) + print(f"FAILED {label}: {exc}") sys.exit(f"failed: {failed}" if failed else None) From 5712d1b99aa94764034b7bf7d3a20f8877cfe5b6 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 12:52:51 +0100 Subject: [PATCH 09/10] test: declare test dependencies inline and run with uv run The dependency list was copied into the README, the test docstring and CI. A PEP 723 header keeps it in one place, like the skill scripts. --- .github/workflows/test.yml | 4 +--- README.md | 5 ++--- tests/test_fetch.py | 14 +++++++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a6bb47..090cf72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,4 @@ jobs: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v6 - name: Unit tests on synthetic stores - run: > - uv run --no-project --python 3.12 --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask - --with fsspec --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests + run: uv run tests/test_fetch.py diff --git a/README.md b/README.md index 43b0b7f..16c1033 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,8 @@ The scripts read remote stores and `uv` installs their packages on first run. Sa ## Tests ```sh -uv run --no-project --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask --with fsspec \ - --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests # offline, synthetic stores -python3 tests/smoke.py # two points from every live store +uv run tests/test_fetch.py # offline, synthetic stores +python3 tests/smoke.py # two points from every live store ``` The offline tests run on every pull request. The live test runs on pull requests that change `skills/` or `tests/`, and every Monday to catch a store that moved. diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 1e40f76..44e879a 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,12 +1,16 @@ -"""Offline tests on synthetic stores. No network. +# /// script +# requires-python = ">=3.12" +# dependencies = ["pytest", "xarray", "zarr>=3", "icechunk", "dask", "fsspec", "aiohttp", "pandas", "pyarrow", "rasterio", "numpy"] +# /// +"""Offline tests on synthetic stores. No network. The dependencies are every skill's, plus pytest. - uv run --no-project --with pytest --with xarray --with "zarr>=3" --with icechunk --with dask --with fsspec \ - --with aiohttp --with pandas --with pyarrow --with rasterio --with numpy pytest tests + uv run tests/test_fetch.py """ import argparse import importlib.util import inspect +import sys from pathlib import Path import numpy as np @@ -132,3 +136,7 @@ def test_cli_takes_one_mode(tmp_path): for wrong in ({"days": None}, {"start": "2023-01-01"}, {"date_col": None}, {"date_col": "sowing"}): with pytest.raises(SystemExit): mod.cli_points(parser, argparse.Namespace(**(args | wrong))) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, *sys.argv[1:]])) From a2ae7da131d181c3d1a4346433c6eea876bf7395 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 12:55:49 +0100 Subject: [PATCH 10/10] refactor(check): follow redirects for any store listed in FOLLOW The CHIRTS-only version guessed between three Zarr layouts and kept hosts in a class attribute. Listing one chunk URL per store lets the check name the download hosts of any store that redirects, such as upcoming copy of Hugging Face copy of SoilGrids. --- scripts/check.py | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/check.py b/scripts/check.py index 00d5573..ebd4c9e 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -8,7 +8,7 @@ import sys import urllib.error import urllib.request -from urllib.parse import urlsplit +from urllib.parse import urljoin, urlsplit # host lists mirror HOSTS in each skill's scripts/fetch.py NEEDS = { @@ -19,18 +19,15 @@ "elevation": ["copernicus-dem-90m.s3.amazonaws.com"], "soilgrids-soil": ["files.isric.org"], } -CHIRTS = "https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr/" -CHUNKS = ["Tmax/0.0.0", "Tmax/0/0/0", "Tmax/c/0/0/0"] # the first chunk under the layouts Zarr uses - - -class Trail(urllib.request.HTTPRedirectHandler): - """Remember each host a download is redirected to.""" +# one data chunk per store that redirects downloads to other hosts, such as Hugging Face; the check follows it to name them +FOLLOW = { + "chirts-temperature": ["https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr/Tmax/c/0/0/0"], +} - hosts: list[str] = [] - def redirect_request(self, req, fp, code, msg, headers, newurl): - self.hosts.append(urlsplit(newurl).hostname) - return super().redirect_request(req, fp, code, msg, headers, newurl) +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args): # hand back each 3xx so its Location can be read + return None def blocked(host: str) -> str | None: @@ -44,19 +41,21 @@ def blocked(host: str) -> str | None: return None -def chirts_download_hosts() -> dict[str, str | None]: - """Hugging Face serves data files from a second host. Follow one chunk to find it and test it.""" - opener = urllib.request.build_opener(Trail) - for chunk in CHUNKS: - Trail.hosts = [] +def redirect_hosts(url: str) -> list[str]: + """Each host a download is sent on to, hop by hop. A host that does not answer ends the chain and is still listed.""" + hosts, opener = [], urllib.request.build_opener(NoRedirect) + for _ in range(5): # real chains are one or two hops; this stops a loop try: - opener.open(urllib.request.Request(CHIRTS + chunk, headers={"Range": "bytes=0-0"}), timeout=20) - return {h: None for h in Trail.hosts} - except urllib.error.HTTPError: - continue # not this layout - except OSError as exc: - return {h: (str(getattr(exc, "reason", exc)) if h == Trail.hosts[-1] else None) for h in Trail.hosts} - return {} + opener.open(urllib.request.Request(url, method="HEAD"), timeout=20) + except urllib.error.HTTPError as exc: + if location := exc.headers.get("Location"): + url = urljoin(url, location) + hosts.append(urlsplit(url).hostname) + continue + except OSError: + pass + break + return list(dict.fromkeys(hosts)) if __name__ == "__main__": @@ -69,8 +68,9 @@ def chirts_download_hosts() -> dict[str, str | None]: failed = [] for need in ["packages", *skills]: results = {h: blocked(h) for h in NEEDS[need]} - if need == "chirts-temperature" and not results["huggingface.co"]: - results |= chirts_download_hosts() + if need in FOLLOW and not any(results.values()): # redirects can only be followed once the store's host answers + for url in FOLLOW[need]: + results |= {h: blocked(h) for h in redirect_hosts(url) if h not in results} for host, why in results.items(): print(f"{'BLOCKED' if why else 'ok':8}{need:20}{host}{f' ({why})' if why else ''}") failed += [host] if why else []