From 6c857c8a051f00aca4dc849b2e8bbab49cc3eaf2 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 15:02:37 +0100 Subject: [PATCH 1/4] feat(chirps): fall back to the CHC server when S3 is blocked Some agent sandboxes block S3 but allow data.chc.ucsb.edu. The yearly NetCDFs there hold the same rnl product, chunked, so point reads stay small. Retries back off on the server's 429 rate limit. --- skills/chirps-rainfall/SKILL.md | 17 +++++++++- skills/chirps-rainfall/scripts/fetch.py | 41 ++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/skills/chirps-rainfall/SKILL.md b/skills/chirps-rainfall/SKILL.md index 9ce6b15..ab0eac0 100644 --- a/skills/chirps-rainfall/SKILL.md +++ b/skills/chirps-rainfall/SKILL.md @@ -20,7 +20,7 @@ uv run scripts/fetch.py points.csv rain.parquet --date-col planting_date --days Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `precip_mm`, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. -The script needs `pypi.org` and `files.pythonhosted.org` on first run and `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` for the data. If S3 is blocked, it switches by itself to the same product on the Climate Hazards Center (CHC) server, `data.chc.ucsb.edu`, and says so. That is the producer's own copy, so values match to about 0.05 mm; report which host was used. The fallback is slower: the server allows a few requests a second, so expect about 20 s per point-year per 10° × 30° tile. If it reports that both hosts are unreachable, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. ## If you need to read the store yourself @@ -38,6 +38,21 @@ ds = xr.open_zarr(icechunk.Repository.open(storage).readonly_session("main").sto For a regional map rather than points, the Hugging Face copy `aaguilar90/chirps-v3-daily-rnl` is cheaper (15° chunks); from R it reads with `terra` and GDAL 3.11+, values × 0.1. +## Fallback: CHC server + +Yearly NetCDF4 files, chunked 31 days × 10° × 30° and gzip-compressed, so xarray reads only the chunks a point needs over HTTP range requests: + +```python +import fsspec, xarray as xr +url = "https://data.chc.ucsb.edu/products/CHIRPS/v3.0/daily/final/rnl/netcdf/byYear/chirps-v3.0.rnl.2024.days_p05.nc" +ds = xr.open_dataset(fsspec.open(url).open(), engine="h5netcdf", chunks={}) +``` + +- Variable `precip` in **mm per day**. Latitude is **ascending** here, unlike the S3 store. +- `rnl` means the pentad totals are split into days using ERA5; `sat` (IMERG, 1998 onward) is a different daily product. The S3 store holds `rnl`. +- The server returns HTTP 429 above a few requests a second. Retry with backoff; `chc_client` in `scripts/fetch.py` shows how. +- Daily Cloud Optimized GeoTIFFs (COGs) are also there under `rnl/cogs/YYYY/`, one file per day. They suit a map for one day, not long point series. + ## Combining with other sources For Tmax and Tmin, pass the same points file and dates to `chirts-temperature`; both have 0.05° resolution; join on `id` and `date` and retain each source's selected cell coordinates. For radiation, humidity, wind or ET use `agera5-climate` (0.1° grid, local-time days); join on `id` and `date`, not cell coordinates. The `climate-hazards` skill is planned and is not bundled here. Return daily data; compute summaries only when requested, using an available workflow with explicit definitions. diff --git a/skills/chirps-rainfall/scripts/fetch.py b/skills/chirps-rainfall/scripts/fetch.py index 321df38..901bcbf 100644 --- a/skills/chirps-rainfall/scripts/fetch.py +++ b/skills/chirps-rainfall/scripts/fetch.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.12" -# dependencies = ["xarray", "zarr>=3", "icechunk", "dask", "pandas", "pyarrow"] +# dependencies = ["xarray", "zarr>=3", "icechunk", "dask", "pandas", "pyarrow", "fsspec", "aiohttp", "aiohttp-retry", "h5netcdf", "h5py"] # /// """Daily CHIRPS v3 rainfall for points. One row per point and day, mm. @@ -8,18 +8,26 @@ uv run fetch.py points.csv out.parquet --date-col planting_date --days 100 points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store. -With --date-col each point gets its own window, from its date (day 0) to --days later.""" +With --date-col each point gets its own window, from its date (day 0) to --days later. +If the S3 store is unreachable, it reads the same data from the CHC server's yearly NetCDF files instead.""" import argparse +import sys import dask import numpy as np +import aiohttp +from aiohttp_retry import ExponentialRetry, RetryClient +import fsspec import icechunk import pandas as pd import xarray as xr STORE = ("dynamical-ucsb-chc-chirps", "ucsb-chc-chirps-analysis-final/v0.1.0.icechunk") HOSTS = ["dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com"] # what a firewall must allow +FALLBACK = ["data.chc.ucsb.edu"] # or this, the producer's own server +CHC = "https://data.chc.ucsb.edu/products/CHIRPS/v3.0/daily/final/rnl/netcdf/byYear/" +NETWORK = (OSError, aiohttp.ClientError, icechunk.StorageError) THREADS = 32 # reads wait on the network, not the CPU @@ -29,6 +37,27 @@ def open_store() -> xr.Dataset: return xr.open_zarr(session.store, consolidated=False, chunks={}) +async def chc_client(**_): + # data.chc.ucsb.edu answers 429 above a few requests a second; back off and retry + retry = ExponentialRetry(attempts=10, start_timeout=0.5, statuses={429, 502, 503, 504}) + return RetryClient(client_session=aiohttp.ClientSession(trust_env=True), retry_options=retry) + + +def open_chc(points: pd.DataFrame, days: int | None) -> xr.Dataset: + """The years the points need from the CHC server's yearly NetCDF files (chunked, so only the needed chunks are read). + + CHIRPS v3 final, ERA5-disaggregated ('rnl'): the product the S3 store holds. Returned in the S3 store's names and units.""" + fs = fsspec.filesystem("https", get_client=chc_client) + day0 = pd.to_datetime(points.date) + last = (day0.max() + pd.Timedelta(days=days)).year if days is not None else pd.Timestamp.now().year + listed = set(fs.ls(CHC, detail=False)) # years not yet published are left out; select_daily reports the gap + urls = [u for y in range(day0.min().year, last + 1) if (u := f"{CHC}chirps-v3.0.rnl.{y}.days_p05.nc") in listed] + if not urls: + raise ValueError(f"no CHIRPS files on {FALLBACK[0]} for {day0.min().year} to {last}") + ds = xr.concat([xr.open_dataset(fs.open(u, block_size=2**20), engine="h5netcdf", chunks={}) for u in urls], "time") + return (ds.precip / 86400).to_dataset(name="precipitation_surface") # mm/day -> kg m-2 s-1, as on S3 + + def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" if missing := {"id", "lat", "lon"} - set(points.columns): @@ -111,8 +140,12 @@ def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None points, days = periods(validate_points(points), start, end, days) try: out = daily(open_store(), points, days) - except (OSError, icechunk.StorageError) as exc: # a block, or a 404 if the store moved; check.py tells which - raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None + except NETWORK as exc: # a block, or a 404 if the store moved; check.py tells which + print(f"could not read {HOSTS[0]} ({exc}); reading {FALLBACK[0]} instead, which is slower", file=sys.stderr) + try: + out = daily(open_chc(points, days), points, days) + except NETWORK as exc: + raise ConnectionError(f"could not read {HOSTS[0]} or {FALLBACK[0]} ({exc}). If the network blocks them, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return out if keep_day else out.drop(columns="day") From 3536d8826e5965ebb455b318759496073be3e221 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 15:02:38 +0100 Subject: [PATCH 2/4] feat(chirts): fall back to the CHC server when Hugging Face is blocked Same reason as CHIRPS: data.chc.ucsb.edu is reachable where Hugging Face is not, and serves the CHIRTS-ERA5 data the Hugging Face store copies. --- skills/chirts-temperature/SKILL.md | 15 +++++++- skills/chirts-temperature/scripts/fetch.py | 43 ++++++++++++++++++++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/skills/chirts-temperature/SKILL.md b/skills/chirts-temperature/SKILL.md index 15510de..f59872a 100644 --- a/skills/chirts-temperature/SKILL.md +++ b/skills/chirts-temperature/SKILL.md @@ -20,7 +20,7 @@ uv run scripts/fetch.py points.csv temp.parquet --date-col planting_date --days Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `tmax_c`, `tmin_c`, `cell_lat`, `cell_lon`. Values above 60 °C are masked (a few source pixels hold impossible values). The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. -The script needs `pypi.org` and `files.pythonhosted.org` on first run and `huggingface.co` and the download host it redirects to (`scripts/check.py` in the repository names it) for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `huggingface.co` and the download host it redirects to (`scripts/check.py` in the repository names it) for the data. If Hugging Face is blocked, it switches by itself to the same product on the Climate Hazards Center (CHC) server, `data.chc.ucsb.edu`, and says so. That is the producer's own copy, so values match to the 0.1 °C rounding of the Hugging Face store; report which host was used. The fallback is slower: the server allows a few requests a second, so expect 30 to 60 s per point-year. If it reports that both hosts are unreachable, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. ## Bandwidth @@ -40,6 +40,19 @@ ds = xr.open_zarr("https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/ - Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. - From R, `terra::rast('ZARR:"path.zarr":/Tmax')` works with GDAL 3.11 or newer, but returns raw integers: multiply by 0.1 and set `NAflag` to -32768. +## Fallback: CHC server + +Yearly NetCDF4 files, one per variable, chunked 19 days × ~7° × ~19° and gzip-compressed, so xarray reads only the chunks a point needs over HTTP range requests: + +```python +import fsspec, xarray as xr +url = "https://data.chc.ucsb.edu/experimental/CHIRTS-ERA5/tmax/netcdf/daily/CHIRTS-ERA5.daily_Tmax.2024.nc" +ds = xr.open_dataset(fsspec.open(url).open(), engine="h5netcdf", chunks={}) +``` + +- Variable `temp` in °C (float, not scaled), dims `latitude`, `longitude`. Tmin is under `tmin/` and `daily_Tmin`. Latitude is **ascending** here. +- The server returns HTTP 429 above a few requests a second. Retry with backoff; `chc_client` in `scripts/fetch.py` shows how. + ## Caveats - CHIRTS-ERA5 is in the Climate Hazards Center's experimental tree, not the validated, station-blended CHIRTS-daily v1.0 (1983 to 2016 only). Don't mix the two. Do not transfer validation claims from CHIRTS-daily to this reconstruction. diff --git a/skills/chirts-temperature/scripts/fetch.py b/skills/chirts-temperature/scripts/fetch.py index 2e3dc6f..5a300e7 100644 --- a/skills/chirts-temperature/scripts/fetch.py +++ b/skills/chirts-temperature/scripts/fetch.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.12" -# dependencies = ["xarray", "zarr>=3", "dask", "fsspec", "aiohttp", "pandas", "pyarrow"] +# dependencies = ["xarray", "zarr>=3", "dask", "fsspec", "aiohttp", "aiohttp-retry", "h5netcdf", "h5py", "pandas", "pyarrow"] # /// """Daily CHIRTS-ERA5 Tmax and Tmin for points. One row per point and day, degrees C. @@ -8,17 +8,25 @@ uv run fetch.py points.csv out.parquet --date-col planting_date --days 100 points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store. -With --date-col each point gets its own window, from its date (day 0) to --days later.""" +With --date-col each point gets its own window, from its date (day 0) to --days later. +If Hugging Face is unreachable, it reads the same data from the CHC server's yearly NetCDF files instead.""" import argparse +import sys +import aiohttp +from aiohttp_retry import ExponentialRetry, RetryClient import dask +import fsspec import numpy as np import pandas as pd import xarray as xr STORE = "https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr" HOSTS = ["huggingface.co"] # what a firewall must allow, plus the download host it redirects to (scripts/check.py names it) +FALLBACK = ["data.chc.ucsb.edu"] # or this, the producer's own server +CHC = "https://data.chc.ucsb.edu/experimental/CHIRTS-ERA5/{v}/netcdf/daily/" +NETWORK = (OSError, aiohttp.ClientError) THREADS = 32 # reads wait on the network, not the CPU TMAX_OK = 60 # a few source pixels hold impossible values; mask above this @@ -28,6 +36,29 @@ def open_store() -> xr.Dataset: return xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"client_kwargs": {"trust_env": True}}) +async def chc_client(**_): + # data.chc.ucsb.edu answers 429 above a few requests a second; back off and retry + retry = ExponentialRetry(attempts=10, start_timeout=0.5, statuses={429, 502, 503, 504}) + return RetryClient(client_session=aiohttp.ClientSession(trust_env=True), retry_options=retry) + + +def open_chc(points: pd.DataFrame, days: int | None) -> xr.Dataset: + """The years the points need from the CHC server's yearly NetCDF files (chunked, so only the needed chunks are read). + + The same CHIRTS-ERA5 the Hugging Face store copies, returned in its names. Only years with both Tmax and Tmin are kept.""" + fs = fsspec.filesystem("https", get_client=chc_client) + day0 = pd.to_datetime(points.date) + last = (day0.max() + pd.Timedelta(days=days)).year if days is not None else pd.Timestamp.now().year + parts = [] + for v, name in (("tmax", "Tmax"), ("tmin", "Tmin")): + listed = set(fs.ls(CHC.format(v=v), detail=False)) # years not yet published are left out; select_daily reports the gap + urls = [u for y in range(day0.min().year, last + 1) if (u := f"{CHC.format(v=v)}CHIRTS-ERA5.daily_{name}.{y}.nc") in listed] + if not urls: + raise ValueError(f"no CHIRTS-ERA5 {name} files on {FALLBACK[0]} for {day0.min().year} to {last}") + parts.append(xr.concat([xr.open_dataset(fs.open(u, block_size=2**20), engine="h5netcdf", chunks={}).temp for u in urls], "time").rename(name)) + return xr.merge(parts, join="inner").rename(latitude="lat", longitude="lon") + + def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" if missing := {"id", "lat", "lon"} - set(points.columns): @@ -110,8 +141,12 @@ def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None points, days = periods(validate_points(points), start, end, days) try: out = daily(open_store(), points, days) - except OSError as exc: # a block, or a 404 if the store moved; check.py tells which - raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None + except NETWORK as exc: # a block, or a 404 if the store moved; check.py tells which + print(f"could not read {HOSTS[0]} ({exc}); reading {FALLBACK[0]} instead, which is slower", file=sys.stderr) + try: + out = daily(open_chc(points, days), points, days) + except NETWORK as exc: + raise ConnectionError(f"could not read {HOSTS[0]} or {FALLBACK[0]} ({exc}). If the network blocks them, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return out if keep_day else out.drop(columns="day") From 8c56111f582f2a4d88233ccd32fdf16c5f900cd6 Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 15:02:38 +0100 Subject: [PATCH 3/4] feat(check): count a skill as blocked only when its fallback is too Otherwise check.py reports CHIRPS and CHIRTS as blocked in sandboxes where their fallback works. --- SKILL.md | 8 ++++---- scripts/check.py | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/SKILL.md b/SKILL.md index 8bb6e40..871146c 100644 --- a/SKILL.md +++ b/SKILL.md @@ -52,17 +52,17 @@ The scripts read remote stores, and `uv` installs their packages on first run, s | Need | Hosts | |---|---| | Packages, first run of any script | `pypi.org`, `files.pythonhosted.org` | -| `chirps-rainfall` | `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` | -| `chirts-temperature` | `huggingface.co` and the download host it redirects to, which `scripts/check.py` names | +| `chirps-rainfall` | `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com`, or else `data.chc.ucsb.edu` | +| `chirts-temperature` | `huggingface.co` and the download host it redirects to, which `scripts/check.py` names, or else `data.chc.ucsb.edu` | | `agera5-climate` | `arco.datastores.ecmwf.int` | | `elevation` | `copernicus-dem-90m.s3.amazonaws.com` | | `soilgrids-soil` | `files.isric.org`; regional 1 km reads also need `huggingface.co` and its download host | -`python3 scripts/check.py [skill ...]` tests them with the standard library only, so it runs where nothing can be installed. When a script cannot read its store, its error names the host and points here. +The CHIRPS and CHIRTS scripts switch to `data.chc.ucsb.edu`, the producer's own server, by themselves when their first host is blocked. That is the same data and needs no action; report which host was used. `python3 scripts/check.py [skill ...]` tests them with the standard library only, so it runs where nothing can be installed. When a script cannot read its store, its error names the host and points here. When a host is blocked: -1. Run `scripts/check.py` once and report the blocked hosts as it prints them. Do not retry with other tools, mirrors or another data source the user did not ask for. +1. Run `scripts/check.py` once and report the blocked hosts as it prints them. Do not retry with other tools, mirrors or another data source the user did not ask for. The built-in CHC fallback is the only exception. 2. Never estimate, simulate or fill in the values. A blocked network is not missing data. 3. Prepare everything that needs no network: the points file and the exact command. 4. Give the user two ways forward: allow the hosts in the agent's network settings (the change may only apply to a new session), or run the command on their own machine and share the output file. diff --git a/scripts/check.py b/scripts/check.py index ebd4c9e..0eb6197 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -19,6 +19,11 @@ "elevation": ["copernicus-dem-90m.s3.amazonaws.com"], "soilgrids-soil": ["files.isric.org"], } +# a second source the fetch script switches to when the first is blocked; the skill works if either is reachable +FALLBACK = { + "chirps-rainfall": ["data.chc.ucsb.edu"], + "chirts-temperature": ["data.chc.ucsb.edu"], +} # one data chunk per store that redirects downloads to other hosts, such as Hugging Face; the check follows it to name them FOLLOW = { "chirts-temperature": ["https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr/Tmax/c/0/0/0"], @@ -71,9 +76,13 @@ def redirect_hosts(url: str) -> list[str]: if need in FOLLOW and not any(results.values()): # redirects can only be followed once the store's host answers for url in FOLLOW[need]: results |= {h: blocked(h) for h in redirect_hosts(url) if h not in results} - for host, why in results.items(): - print(f"{'BLOCKED' if why else 'ok':8}{need:20}{host}{f' ({why})' if why else ''}") - failed += [host] if why else [] + backup = {h: blocked(h) for h in FALLBACK.get(need, [])} + for host, why in (results | backup).items(): + print(f"{'BLOCKED' if why else 'ok':8}{need:20}{host}{' (fallback)' if host in backup else ''}{f' ({why})' if why else ''}") + if any(results.values()) and (not backup or any(backup.values())): + failed += [h for h, why in (results | backup).items() if why] + elif any(results.values()): + print(f"{'':8}{need:20}will use the fallback, which is slower") if failed: print(f"\nThis network blocks {len(failed)} host(s). Allow them, or run the fetch on a machine with open internet:") sys.exit("\n".join(failed)) From ec73bb1f23cb36ef9ef3a5542eb66968429fc91c Mon Sep 17 00:00:00 2001 From: Brayden Youngberg Date: Wed, 18 Nov 2026 15:02:38 +0100 Subject: [PATCH 4/4] test: cover the CHC fallback and keep its helpers in sync --- tests/test_fetch.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 44e879a..d032501 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ # /// script # requires-python = ">=3.12" -# dependencies = ["pytest", "xarray", "zarr>=3", "icechunk", "dask", "fsspec", "aiohttp", "pandas", "pyarrow", "rasterio", "numpy"] +# dependencies = ["pytest", "xarray", "zarr>=3", "icechunk", "dask", "fsspec", "aiohttp", "aiohttp-retry", "h5netcdf", "h5py", "pandas", "pyarrow", "rasterio", "numpy"] # /// """Offline tests on synthetic stores. No network. The dependencies are every skill's, plus pytest. @@ -63,7 +63,7 @@ def weather(request, monkeypatch): def test_shared_helpers_are_identical(): """Each skill is self-contained, so the helpers are copied. This catches a copy that drifts.""" - shared = {"validate_points": SKILLS, "select_daily": WEATHER, "periods": WEATHER, "cli_points": WEATHER} + shared = {"validate_points": SKILLS, "select_daily": WEATHER, "periods": WEATHER, "cli_points": WEATHER, "chc_client": ["chirps-rainfall", "chirts-temperature"]} for name, skills in shared.items(): assert len({inspect.getsource(getattr(MODS[s], name)) for s in skills}) == 1, name @@ -71,6 +71,7 @@ def test_shared_helpers_are_identical(): def test_check_knows_every_host(): check = load(ROOT / "scripts" / "check.py") assert {s: check.NEEDS[s] for s in SKILLS} == {s: MODS[s].HOSTS for s in SKILLS} + assert check.FALLBACK == {s: m.FALLBACK for s, m in MODS.items() if hasattr(m, "FALLBACK")} def test_window_per_point(weather): @@ -103,10 +104,20 @@ def test_point_on_a_cell_edge_is_on_grid(weather): def test_a_failed_read_points_to_check(weather, monkeypatch): mod, _, _ = weather monkeypatch.setattr(mod, "open_store", lambda: (_ for _ in ()).throw(OSError("Cannot connect to host 127.0.0.1:9"))) + monkeypatch.setattr(mod, "open_chc", lambda *_: (_ for _ in ()).throw(OSError("Cannot connect to host 127.0.0.1:9")), raising=False) with pytest.raises(ConnectionError, match="check.py"): mod.fetch(POINTS, "2023-03-01", "2023-03-02") +@pytest.mark.parametrize("skill", ["chirps-rainfall", "chirts-temperature"]) +def test_a_blocked_store_falls_back_to_chc(skill, monkeypatch): + mod, (lat, lon, variables, descending, column, scale) = MODS[skill], STORES[skill] + monkeypatch.setattr(mod, "open_store", lambda: (_ for _ in ()).throw(OSError("blocked"))) + monkeypatch.setattr(mod, "open_chc", lambda *_: store(lat, lon, variables, False)) # CHC files run south to north + out = mod.fetch(POINTS, "2023-03-01", "2023-03-05") + assert out[out.id == "a"][column].iloc[0] == pytest.approx(6.0 * scale, rel=1e-5) + + def test_window_outside_the_store_fails(weather): mod, _, _ = weather late = POINTS.assign(date="2023-12-28")