diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml new file mode 100644 index 0000000..79ba003 --- /dev/null +++ b/.github/workflows/live.yml @@ -0,0 +1,25 @@ +name: live + +on: + pull_request: + paths: ["skills/**", "tests/**"] + push: + branches: [main] + paths: ["skills/**", "tests/**"] + schedule: + - cron: "23 5 * * 1" # Mondays, to catch a data store that moved or changed + workflow_dispatch: + +permissions: + contents: read + +jobs: + live: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Two points from every store + run: python3 tests/smoke.py + env: + CDSAPI_KEY: ${{ secrets.CDSAPI_KEY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..090cf72 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,18 @@ +name: test + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + offline: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - name: Unit tests on synthetic stores + run: uv run tests/test_fetch.py diff --git a/AGENTS.md b/AGENTS.md index 04cf10d..5648c8d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This repository holds skills that fetch weather, elevation and soil data for poi 1. Read `SKILL.md` first. It says which skill answers which need and the rules every skill shares. 2. Open only the skill folder the question needs, under `skills/`. Each `SKILL.md` is self-contained and its `scripts/` file does the fetch. -3. Run scripts with `uv run`. Do not install anything else without the user's agreement. +3. Run scripts with `uv run`. Do not install anything else without the user's agreement. If a script cannot reach its data store, or `uv` cannot install packages, follow "Network" in `SKILL.md`: run `python3 scripts/check.py`, name the blocked hosts, and hand the user the exact command. Never estimate values or switch source. 4. Keep the data's caveats with the data. Say which source, grid and dates a result came from. 5. Never invent a DOI, licence or date. Each skill ends with what to cite. 6. Write maniva and tricot in lowercase, always, including headings, table cells and the start of a sentence. Never write "Maniva" or "Tricot". tricot stands for triadic comparison of technology options. Quote published titles exactly as printed, even where they capitalise tricot or expand it as "triadic comparisons of technologies". diff --git a/README.md b/README.md index 4bed01f..16c1033 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,27 @@ ln -s "$PWD"/environment-data/skills/* ~/.claude/skills/ **Codex and Gemini CLI.** Same, into `~/.agents/skills/` or `~/.gemini/skills/`. -**Without an agent.** Run any script directly, e.g. `uv run skills/chirps-rainfall/scripts/fetch.py points.csv rain.parquet --start 2020-01-01`. +**Without an agent.** Run any script directly, e.g. `uv run skills/chirps-rainfall/scripts/fetch.py points.csv rain.parquet --start 2020-01-01`. When each point has its own date, such as a planting date, fetch a window per point: `--date-col planting_date --days 100`. **R users.** Run the scripts as above, then read the parquet outputs in R with `nanoparquet::read_parquet()`, a small package with no dependencies, or `arrow::read_parquet()` if you already use arrow. --- +## Network + +The scripts read remote stores and `uv` installs their packages on first run. Sandboxed agents, such as a cloud chat session, and some institutional networks block those hosts. `python3 scripts/check.py` lists what is blocked, using the standard library only. The hosts are in [SKILL.md](SKILL.md#network), with what an agent does when they are out of reach. + +## Tests + +```sh +uv run tests/test_fetch.py # offline, synthetic stores +python3 tests/smoke.py # two points from every live store +``` + +The offline tests run on every pull request. The live test runs on pull requests that change `skills/` or `tests/`, and every Monday to catch a store that moved. + +--- + ## Cite Cite with [CITATION.cff](CITATION.cff). diff --git a/SKILL.md b/SKILL.md index ea7b7b0..ce7ded6 100644 --- a/SKILL.md +++ b/SKILL.md @@ -26,22 +26,53 @@ Write maniva and tricot in lowercase, always. That holds in headings, table cell ## Three ways to use a skill -1. **Run the script.** Resolve `scripts/fetch.py` from the selected skill directory and follow its CLI example (weather also requires `--start`). Use this by default. Output is parquet; name the output `.csv` for CSV, which suits elevation, soil and small weather pulls (daily weather for many points gets large). -2. **Import the script module.** Its `fetch()` takes a pandas data frame with `id`, `lat`, `lon` and returns a data frame. Use it inside a larger Python pipeline. +1. **Run the script.** Resolve `scripts/fetch.py` from the selected skill directory and follow its CLI example (weather also requires a period, see "Dates"). Use this by default. Output is parquet; name the output `.csv` for CSV, which suits elevation, soil and small weather pulls (daily weather for many points gets large). +2. **Import the script module.** Its `fetch()` takes a pandas data frame with `id`, `lat`, `lon` (and `date` for a window per point) and returns a data frame. Use it inside a larger Python pipeline. 3. **Write your own reader.** Each skill's "If you need to read ... yourself" section gives the store path and the traps (units, scaling, projection, grid order). Use it for R, other tools or needs the script does not cover. ## Points Users give points in any form: a table with other column names, Excel, tricot data, or coordinates typed in chat. Build the `id`, `lat`, `lon` table yourself, using an existing identifier as `id` or the row number if there is none. Tell the user which columns you used. Use WGS84 decimal degrees with `lat` within −90 to 90 and `lon` within −180 to 180. Preserve IDs as strings and require unique, non-empty IDs for unambiguous joins. Investigate suspected swapped coordinates; never guess a correction or shift points to obtain values. Convert known missing-coordinate sentinels to blanks. Leave rows with missing coordinates in; the scripts return them as NaN. -For a tricot-data trial file, take `site_data[]`: `site_id` → `id`, `latitude` → `lat`, `longitude` → `lon`. Some sites have null coordinates; keep them as blanks. tricot site coordinates are displaced up to 1.5 km for privacy (`metadata.location_privacy`), and some are filled from the mean of their district (`coordinate_source`, `coordinate_uncertainty_m`). That is fine for 5 to 10 km weather grids. For elevation (90 m) and soil (250 m), values describe the published point, not the farm; say so when you report them. +For a tricot-data trial file, take `site_data[]`: `site_id` → `id`, `latitude` → `lat`, `longitude` → `lon`, and keep `planting_date` for a window per site (`--date-col planting_date`). Some sites have null coordinates or planting dates; keep them as blanks. Other trial tables follow the same pattern: one row per plot or block, its identifier as `id`, its coordinates and its planting date. tricot site coordinates are displaced up to 1.5 km for privacy (`metadata.location_privacy`), and some are filled from the mean of their district (`coordinate_source`, `coordinate_uncertainty_m`). That is fine for 5 to 10 km weather grids. For elevation (90 m) and soil (250 m), values describe the published point, not the farm; say so when you report them. + +## Dates + +The weather skills take a period in one of two ways. Use the same one for every source you combine. + +- **One period for all points.** `--start 2020-01-01 --end 2020-12-31`. Use it for a season, a year or a climatology. +- **A window per point.** `--date-col planting_date --days 100` fetches each point from its own date (day 0) to 100 days later, 101 rows per point, with a `day` column counting from 0. Use it whenever points have their own dates, such as planting dates per plot or survey dates per household. Points planted weeks or years apart then cost only their own windows, not the whole span between the first and the last. + +Keep the date column in the points file under its own name and pass that name to `--date-col`. Dates are ISO (`YYYY-MM-DD`). A point with a blank date gets no rows and the script lists it; report those IDs. A window that runs past the store's last day fails rather than being shortened. Ask the user for the number of days if they did not give one; the crop's season length is theirs to state. + +## Network + +The scripts read remote stores, and `uv` installs their packages on first run, so they need these hosts. Sandboxed agents and institutional networks often block them. + +| Need | Hosts | +|---|---| +| Packages, first run of any script | `pypi.org`, `files.pythonhosted.org` | +| `chirps-rainfall` | `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` | +| `chirts-temperature` | `huggingface.co` and the download host it redirects to, which `scripts/check.py` names | +| `agera5-climate` | `arco.datastores.ecmwf.int` | +| `elevation` | `copernicus-dem-90m.s3.amazonaws.com` | +| `soilgrids-soil` | `files.isric.org` | + +`python3 scripts/check.py [skill ...]` tests them with the standard library only, so it runs where nothing can be installed. When a script cannot read its store, its error names the host and points here. + +When a host is blocked: + +1. Run `scripts/check.py` once and report the blocked hosts as it prints them. Do not retry with other tools, mirrors or another data source the user did not ask for. +2. Never estimate, simulate or fill in the values. A blocked network is not missing data. +3. Prepare everything that needs no network: the points file and the exact command. +4. Give the user two ways forward: allow the hosts in the agent's network settings (the change may only apply to a new session), or run the command on their own machine and share the output file. ## Rules 1. Open the one skill the question needs. Each is self-contained. Pass the same points and dates between skills. 2. To run a script, use `uv run`. It needs only `uv`, which fetches Python and the packages on first run. If `uv` is missing, give the user the one-line install from https://docs.astral.sh/uv/getting-started/installation/ and ask before installing. 3. Return the output path, source/store, grid, units, actual dates (or soil depths/statistic), retrieval date and relevant caveats. Values are gridded estimates for a cell, not station readings for a point. -4. Report points and variables with missing values, including partial date gaps. Missing coordinates, coverage limits and source masks are different from network or authentication failures; do not label every failure as water or fill gaps silently. +4. Report points and variables with missing values, including partial date gaps. Missing coordinates, coverage limits and source masks are different from network or authentication failures; do not label every failure as water or fill gaps silently. For a network failure follow "Network". 5. Return daily data when a calling skill asked for it. Start hazard analysis only when the user asked for it. 6. Never invent a DOI, licence or date. Each skill ends with what to cite. diff --git a/llms.txt b/llms.txt index e209777..cb27fca 100644 --- a/llms.txt +++ b/llms.txt @@ -11,6 +11,7 @@ Read SKILL.md first. It says which skill answers which need and the rules they s - [SKILL.md](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/SKILL.md): index of skills and shared rules - [AGENTS.md](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/AGENTS.md): short instructions for AI agents - [README.md](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/README.md): sources, install and licence +- [scripts/check.py](https://raw.githubusercontent.com/AgrDataSci/environment-data/main/scripts/check.py): tests whether this network reaches the data stores; standard library only ## Skills diff --git a/scripts/check.py b/scripts/check.py new file mode 100644 index 0000000..ebd4c9e --- /dev/null +++ b/scripts/check.py @@ -0,0 +1,80 @@ +"""Can this machine reach what the skills need? Standard library only, so it runs where packages cannot be installed. + + python3 scripts/check.py [skill ...] + +Prints one line per host and exits 1 if any is blocked. Run it first in a sandbox, or after a fetch fails to connect.""" + +import shutil +import sys +import urllib.error +import urllib.request +from urllib.parse import urljoin, urlsplit + +# host lists mirror HOSTS in each skill's scripts/fetch.py +NEEDS = { + "packages": ["pypi.org", "files.pythonhosted.org"], # uv installs the Python packages from here on first run + "chirps-rainfall": ["dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com"], + "chirts-temperature": ["huggingface.co"], + "agera5-climate": ["arco.datastores.ecmwf.int"], + "elevation": ["copernicus-dem-90m.s3.amazonaws.com"], + "soilgrids-soil": ["files.isric.org"], +} +# one data chunk per store that redirects downloads to other hosts, such as Hugging Face; the check follows it to name them +FOLLOW = { + "chirts-temperature": ["https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr/Tmax/c/0/0/0"], +} + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args): # hand back each 3xx so its Location can be read + return None + + +def blocked(host: str) -> str | None: + """Why the host is out of reach, or None. Any HTTP answer, even 403 or 404, means the host is reachable.""" + try: + urllib.request.urlopen(urllib.request.Request(f"https://{host}/", method="HEAD"), timeout=20) + except urllib.error.HTTPError: + return None + except OSError as exc: + return str(getattr(exc, "reason", exc)) + return None + + +def redirect_hosts(url: str) -> list[str]: + """Each host a download is sent on to, hop by hop. A host that does not answer ends the chain and is still listed.""" + hosts, opener = [], urllib.request.build_opener(NoRedirect) + for _ in range(5): # real chains are one or two hops; this stops a loop + try: + opener.open(urllib.request.Request(url, method="HEAD"), timeout=20) + except urllib.error.HTTPError as exc: + if location := exc.headers.get("Location"): + url = urljoin(url, location) + hosts.append(urlsplit(url).hostname) + continue + except OSError: + pass + break + return list(dict.fromkeys(hosts)) + + +if __name__ == "__main__": + skills = sys.argv[1:] or [s for s in NEEDS if s != "packages"] + if unknown := set(skills) - set(NEEDS): + sys.exit(f"unknown skill {sorted(unknown)}; choose from {', '.join(s for s in NEEDS if s != 'packages')}") + print(f"uv: {shutil.which('uv') or 'not found, see https://docs.astral.sh/uv/getting-started/installation/'}") + if sys.version_info < (3, 12): + print(f"python {sys.version.split()[0]}: older than 3.12, so uv will also download Python from github.com") + failed = [] + for need in ["packages", *skills]: + results = {h: blocked(h) for h in NEEDS[need]} + if need in FOLLOW and not any(results.values()): # redirects can only be followed once the store's host answers + for url in FOLLOW[need]: + results |= {h: blocked(h) for h in redirect_hosts(url) if h not in results} + for host, why in results.items(): + print(f"{'BLOCKED' if why else 'ok':8}{need:20}{host}{f' ({why})' if why else ''}") + failed += [host] if why else [] + if failed: + print(f"\nThis network blocks {len(failed)} host(s). Allow them, or run the fetch on a machine with open internet:") + sys.exit("\n".join(failed)) + print("\nAll hosts reachable.") diff --git a/skills/agera5-climate/SKILL.md b/skills/agera5-climate/SKILL.md index 62f4877..a5c8ccf 100644 --- a/skills/agera5-climate/SKILL.md +++ b/skills/agera5-climate/SKILL.md @@ -15,9 +15,10 @@ Resolve `scripts/fetch.py` relative to this skill's directory, not the user's wo ```sh uv run scripts/fetch.py points.csv weather.parquet --start 2020-01-01 --end 2025-12-31 +uv run scripts/fetch.py points.csv weather.parquet --date-col planting_date --days 100 ``` -`--start` is required; use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)) and a key (below). Output is one row per point and day: `id`, `date`, the variables, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs without moving the points. +Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)) and a key (below). Output is one row per point and day: `id`, `date`, the variables, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs without moving the points. Default variables are the ones CHIRPS and CHIRTS don't have. Pick others with `--vars`: @@ -46,6 +47,8 @@ The script looks for `CDSAPI_KEY` in the environment, then the `key:` line of `~ key: ``` +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `arco.datastores.ecmwf.int` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. + ## If you need to read the store yourself ```python @@ -57,7 +60,7 @@ ds = xr.open_zarr("https://arco.datastores.ecmwf.int/cadl-arco-geo-001/arco/sis_ - 27 variables, `ds.data_vars` lists them. Temperatures in K, radiation in J m⁻² d⁻¹, missing is `-9999` and decodes to NaN. - Latitude ascending from -90, longitude from -180. Chunks are 2112 days × 16 × 32 cells, so long periods at a few places are cheap and one day over a region is slow. - Each day runs midnight to midnight local time, not UTC. -- Select variables and dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. +- Select variables and dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. ## Caveats diff --git a/skills/agera5-climate/scripts/fetch.py b/skills/agera5-climate/scripts/fetch.py index d5b41f4..be1744f 100644 --- a/skills/agera5-climate/scripts/fetch.py +++ b/skills/agera5-climate/scripts/fetch.py @@ -5,8 +5,10 @@ """Daily AgERA5 weather for points. One row per point and day. uv run fetch.py points.csv weather.parquet --start 2020-01-01 [--end 2025-12-31] [--vars srad,rhmax,rhmin,wind,et0,vpd] + uv run fetch.py points.csv weather.parquet --date-col planting_date --days 100 points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. Needs a Copernicus CDS key in CDSAPI_KEY or ~/.cdsapirc. +With --date-col each point gets its own window, from its date (day 0) to --days later. Default variables are the ones CHIRPS and CHIRTS do not have; the rest are available by name.""" import argparse @@ -14,10 +16,12 @@ from pathlib import Path import dask +import numpy as np import pandas as pd import xarray as xr STORE = "https://arco.datastores.ecmwf.int/cadl-arco-geo-001/arco/sis_agrometeorological_indicators/all/geoChunked.zarr" +HOSTS = ["arco.datastores.ecmwf.int"] # what a firewall must allow THREADS = 32 # reads wait on the network, not the CPU # friendly name -> (store variable, scale, offset) so that value = store * scale + offset VARS = { @@ -47,9 +51,18 @@ def cds_key() -> str: raise SystemExit("No CDS key. Set CDSAPI_KEY or write ~/.cdsapirc; see SKILL.md for how to get one.") +def open_store() -> xr.Dataset: + # trust_env makes the HTTP client honour HTTPS_PROXY, which sandboxes and institutional networks route through + options = {"headers": {"Authorization": f"Bearer {cds_key()}"}, "client_kwargs": {"trust_env": True}} + return xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options=options) + + def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -58,50 +71,98 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: return points -def select_daily(ds, points, start, end, lat, lon): - """Nearest cell per point. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" - first, last = pd.to_datetime(ds.time.values[[0, -1]]) - start, end = pd.Timestamp(start), pd.Timestamp(end or last) - if not first <= start <= end <= last: # else the store silently returns a shorter series - raise ValueError(f"dates must be ordered and within {first.date()} to {last.date()}") - on_grid = points.lat.notna() & points.lon.notna() - for col, name in (("lat", lat), ("lon", lon)): - axis = ds[name].values - half = abs(axis[1] - axis[0]) / 2 - on_grid &= points[col].between(axis.min() - half, axis.max() + half) - pts = points[on_grid] - sel = {name: xr.DataArray(pts[col].values, dims="pt", coords={"pt": pts.index}) for col, name in (("lat", lat), ("lon", lon))} +def periods(points: pd.DataFrame, start, end, days) -> tuple[pd.DataFrame, int | None]: + """Each point's day 0 in `date`, and the days after it: one period for all (start, end), or each point's own date and days.""" + if days is None: + if start is None: + raise ValueError("give start, or days with a date column in points") + days = (pd.Timestamp(end) - pd.Timestamp(start)).days if end else None # None: to the store's last day + return points.assign(date=start), days + if "date" not in points: + raise ValueError("a window per point needs a date column") + points = points[pd.to_datetime(points.date).notna()].reset_index(drop=True) # no date, no window + if points.empty: + raise ValueError("no point has a date") + return points, days + + +def select_daily(ds, points, days, lat, lon): + """Nearest cell per point, from its `date` (day 0) to `days` later, or to the store's last day if days is None. + + One vectorised read, so dask fetches each chunk once and in parallel. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" + times = ds.indexes["time"].normalize() + day0 = pd.to_datetime(points.date).dt.normalize().values + days = (times[-1] - day0.min()).days if days is None else days + want = day0[:, None] + np.arange(max(days, 0) + 1) * np.timedelta64(1, "D") + ti = times.get_indexer(want.ravel()).reshape(want.shape) + if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series + raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " + f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") + idx = {} + for col, name in (("lat", lat), ("lon", lon)): # -1 when off grid or missing; it reads the last cell, then is masked + half = abs(float(ds[name][1] - ds[name][0])) / 2 + 1e-6 # slack: points on a cell edge must not round off grid + near = ds.indexes[name].get_indexer(points[col], method="nearest", tolerance=half) + idx[name] = xr.DataArray(near, dims="pt", coords={"pt": points.id.values}) + one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index + idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): - sub = ds.sel(time=slice(start, end)).sel(**sel, method="nearest").compute() - return sub.reindex(pt=points.index) + sub = ds.isel(idx).compute() + if one: + sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") + keep = (idx[lat] >= 0) & (idx[lon] >= 0) + return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) -def fetch(points: pd.DataFrame, start: str, end: str | None = None, names: list[str] | None = None) -> pd.DataFrame: - points = validate_points(points) - names = DEFAULT.split(",") if names is None else names - if bad := set(names) - set(VARS): - raise ValueError(f"unknown variables {sorted(bad)}; choose from {', '.join(VARS)}") - ds = xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"headers": {"Authorization": f"Bearer {cds_key()}"}, "client_kwargs": {"trust_env": True}}) # trust_env: honour HTTPS_PROXY - store_vars = [VARS[n][0] for n in names] - sub = select_daily(ds[store_vars], points, start, end, "latitude", "longitude") - df = sub.to_dataframe().reset_index().rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) +def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: + """One period for all points (--start), or a window per point (--date-col with --days).""" + if bool(a.start) == bool(a.date_col) or (a.date_col and (a.days is None or a.end)): + p.error("give --start [--end] for one period, or --date-col with --days for a window per point") + points = pd.read_csv(a.points, dtype={"id": str}) + if a.date_col and a.date_col != "date": + if a.date_col not in points: + p.error(f"no column {a.date_col} in {a.points}") + points = points.drop(columns="date", errors="ignore").rename(columns={a.date_col: "date"}) + return points + + +def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None, names: list[str]) -> pd.DataFrame: + sub = select_daily(ds[[VARS[n][0] for n in names]], points, days, "latitude", "longitude") + df = sub.to_dataframe().reset_index().rename(columns={"pt": "id", "latitude": "cell_lat", "longitude": "cell_lon"}) for n in names: var, scale, offset = VARS[n] df[n] = (df.pop(var) * scale + offset).astype("float32") - df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date - return df[["id", "date", *names, "cell_lat", "cell_lon"]] + return df[["id", "date", "day", *names, "cell_lat", "cell_lon"]] + + +def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, names: list[str] | None = None, days: int | None = None) -> pd.DataFrame: + """One period for all points (start, end), or with days a window per point from its `date` column.""" + names = DEFAULT.split(",") if names is None else names + if bad := set(names) - set(VARS): + raise ValueError(f"unknown variables {sorted(bad)}; choose from {', '.join(VARS)}") + keep_day = days is not None + points, days = periods(validate_points(points), start, end, days) + try: + out = daily(open_store(), points, days, names) + except OSError as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None + return out if keep_day else out.drop(columns="day") if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("points"), p.add_argument("out"), p.add_argument("--start", required=True), p.add_argument("--end") + p.add_argument("points"), p.add_argument("out"), p.add_argument("--start"), p.add_argument("--end") + p.add_argument("--date-col", help="column with one date per point, such as the planting date") + p.add_argument("--days", type=int, help="days to fetch after each point's date; 100 gives days 0 to 100") p.add_argument("--vars", default=DEFAULT, help=f"comma-separated from {', '.join(VARS)}") a = p.parse_args() names = a.vars.split(",") - out = fetch(pd.read_csv(a.points, dtype={"id": str}), a.start, a.end, names) + points = cli_points(p, a) + out = fetch(points, a.start, a.end, names, a.days) out.to_csv(a.out, index=False) if a.out.endswith(".csv") else out.to_parquet(a.out, index=False, compression="zstd") print(f"{out.id.nunique()} points, {out.date.min()} to {out.date.max()}, {len(out):,} rows, {names} -> {a.out}") miss = out[names].isna().all(axis=1).groupby(out.id).all() if miss.any(): print(f"all-NaN: {list(miss[miss].index)}") + if nodate := sorted(set(points.id) - set(out.id)): + print(f"no date, so no window fetched: {nodate}") diff --git a/skills/chirps-rainfall/SKILL.md b/skills/chirps-rainfall/SKILL.md index c3de7c6..9ce6b15 100644 --- a/skills/chirps-rainfall/SKILL.md +++ b/skills/chirps-rainfall/SKILL.md @@ -15,9 +15,12 @@ Resolve `scripts/fetch.py` relative to this skill's directory, not the user's wo ```sh uv run scripts/fetch.py points.csv rain.parquet --start 2020-01-01 --end 2025-12-31 +uv run scripts/fetch.py points.csv rain.parquet --date-col planting_date --days 100 ``` -`--start` is required; use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `precip_mm`, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. +Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `precip_mm`, `cell_lat`, `cell_lon`. The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. + +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. ## If you need to read the store yourself @@ -31,7 +34,7 @@ ds = xr.open_zarr(icechunk.Repository.open(storage).readonly_session("main").sto - One variable, `precipitation_surface`, in **kg m⁻² s⁻¹**. Multiply by 86400 for mm per day. Dims `(time, latitude, longitude)`. - Latitude is descending. Slice north to south: `latitude=slice(north, south)`. - Land only, 60°S to 60°N. Sea cells are NaN, so check coastal points after fetching. -- Chunks are 365 days × 3° × 3°, so a point read costs one small request per year per 3° tile. Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, and raise dask's thread count (reads are network-bound). Cost scales with the number of 3° tiles the points cover, not the number of points. +- Chunks are 365 days × 3° × 3°, so a point read costs one small request per year per 3° tile. Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, and raise dask's thread count (reads are network-bound). Cost scales with the number of 3° tiles the points cover, not the number of points. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. For a regional map rather than points, the Hugging Face copy `aaguilar90/chirps-v3-daily-rnl` is cheaper (15° chunks); from R it reads with `terra` and GDAL 3.11+, values × 0.1. diff --git a/skills/chirps-rainfall/scripts/fetch.py b/skills/chirps-rainfall/scripts/fetch.py index 718e174..321df38 100644 --- a/skills/chirps-rainfall/scripts/fetch.py +++ b/skills/chirps-rainfall/scripts/fetch.py @@ -5,16 +5,21 @@ """Daily CHIRPS v3 rainfall for points. One row per point and day, mm. uv run fetch.py points.csv out.parquet --start 2020-01-01 [--end 2025-12-31] + uv run fetch.py points.csv out.parquet --date-col planting_date --days 100 -points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store.""" +points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store. +With --date-col each point gets its own window, from its date (day 0) to --days later.""" import argparse + import dask +import numpy as np import icechunk import pandas as pd import xarray as xr STORE = ("dynamical-ucsb-chc-chirps", "ucsb-chc-chirps-analysis-final/v0.1.0.icechunk") +HOSTS = ["dynamical-ucsb-chc-chirps.s3.us-west-2.amazonaws.com"] # what a firewall must allow THREADS = 32 # reads wait on the network, not the CPU @@ -26,7 +31,10 @@ def open_store() -> xr.Dataset: def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -35,42 +43,91 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: return points -def select_daily(ds, points, start, end, lat, lon): - """Nearest cell per point. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" - first, last = pd.to_datetime(ds.time.values[[0, -1]]) - start, end = pd.Timestamp(start), pd.Timestamp(end or last) - if not first <= start <= end <= last: # else the store silently returns a shorter series - raise ValueError(f"dates must be ordered and within {first.date()} to {last.date()}") - on_grid = points.lat.notna() & points.lon.notna() - for col, name in (("lat", lat), ("lon", lon)): - axis = ds[name].values - half = abs(axis[1] - axis[0]) / 2 - on_grid &= points[col].between(axis.min() - half, axis.max() + half) - pts = points[on_grid] - sel = {name: xr.DataArray(pts[col].values, dims="pt", coords={"pt": pts.index}) for col, name in (("lat", lat), ("lon", lon))} +def periods(points: pd.DataFrame, start, end, days) -> tuple[pd.DataFrame, int | None]: + """Each point's day 0 in `date`, and the days after it: one period for all (start, end), or each point's own date and days.""" + if days is None: + if start is None: + raise ValueError("give start, or days with a date column in points") + days = (pd.Timestamp(end) - pd.Timestamp(start)).days if end else None # None: to the store's last day + return points.assign(date=start), days + if "date" not in points: + raise ValueError("a window per point needs a date column") + points = points[pd.to_datetime(points.date).notna()].reset_index(drop=True) # no date, no window + if points.empty: + raise ValueError("no point has a date") + return points, days + + +def select_daily(ds, points, days, lat, lon): + """Nearest cell per point, from its `date` (day 0) to `days` later, or to the store's last day if days is None. + + One vectorised read, so dask fetches each chunk once and in parallel. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" + times = ds.indexes["time"].normalize() + day0 = pd.to_datetime(points.date).dt.normalize().values + days = (times[-1] - day0.min()).days if days is None else days + want = day0[:, None] + np.arange(max(days, 0) + 1) * np.timedelta64(1, "D") + ti = times.get_indexer(want.ravel()).reshape(want.shape) + if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series + raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " + f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") + idx = {} + for col, name in (("lat", lat), ("lon", lon)): # -1 when off grid or missing; it reads the last cell, then is masked + half = abs(float(ds[name][1] - ds[name][0])) / 2 + 1e-6 # slack: points on a cell edge must not round off grid + near = ds.indexes[name].get_indexer(points[col], method="nearest", tolerance=half) + idx[name] = xr.DataArray(near, dims="pt", coords={"pt": points.id.values}) + one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index + idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): - sub = ds.sel(time=slice(start, end)).sel(**sel, method="nearest").compute() - return sub.reindex(pt=points.index) + sub = ds.isel(idx).compute() + if one: + sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") + keep = (idx[lat] >= 0) & (idx[lon] >= 0) + return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) -def fetch(points: pd.DataFrame, start: str, end: str | None = None) -> pd.DataFrame: - points = validate_points(points) - ds = open_store() - sub = select_daily(ds.precipitation_surface, points, start, end, "latitude", "longitude") +def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: + """One period for all points (--start), or a window per point (--date-col with --days).""" + if bool(a.start) == bool(a.date_col) or (a.date_col and (a.days is None or a.end)): + p.error("give --start [--end] for one period, or --date-col with --days for a window per point") + points = pd.read_csv(a.points, dtype={"id": str}) + if a.date_col and a.date_col != "date": + if a.date_col not in points: + p.error(f"no column {a.date_col} in {a.points}") + points = points.drop(columns="date", errors="ignore").rename(columns={a.date_col: "date"}) + return points + + +def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None) -> pd.DataFrame: + sub = select_daily(ds.precipitation_surface, points, days, "latitude", "longitude") df = (sub * 86400).to_dataframe(name="precip_mm").reset_index() # kg m-2 s-1 -> mm/day - df["id"] = points.id.values[df.pt] df["date"] = df.time.dt.date - df = df.rename(columns={"latitude": "cell_lat", "longitude": "cell_lon"}) - return df[["id", "date", "precip_mm", "cell_lat", "cell_lon"]].astype({"precip_mm": "float32"}) + df = df.rename(columns={"pt": "id", "latitude": "cell_lat", "longitude": "cell_lon"}) + return df[["id", "date", "day", "precip_mm", "cell_lat", "cell_lon"]].astype({"precip_mm": "float32"}) + + +def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, days: int | None = None) -> pd.DataFrame: + """One period for all points (start, end), or with days a window per point from its `date` column.""" + keep_day = days is not None + points, days = periods(validate_points(points), start, end, days) + try: + out = daily(open_store(), points, days) + except (OSError, icechunk.StorageError) as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None + return out if keep_day else out.drop(columns="day") if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("points"), p.add_argument("out"), p.add_argument("--start", required=True), p.add_argument("--end") + p.add_argument("points"), p.add_argument("out"), p.add_argument("--start"), p.add_argument("--end") + p.add_argument("--date-col", help="column with one date per point, such as the planting date") + p.add_argument("--days", type=int, help="days to fetch after each point's date; 100 gives days 0 to 100") a = p.parse_args() - out = fetch(pd.read_csv(a.points, dtype={"id": str}), a.start, a.end) + points = cli_points(p, a) + out = fetch(points, a.start, a.end, a.days) out.to_csv(a.out, index=False) if a.out.endswith(".csv") else out.to_parquet(a.out, index=False, compression="zstd") sea = out.groupby("id").precip_mm.apply(lambda s: s.isna().all()) print(f"{out.id.nunique()} points, {out.date.min()} to {out.date.max()}, {len(out):,} rows -> {a.out}") if sea.any(): print(f"all-NaN (missing coordinates, outside coverage or source nodata): {list(sea[sea].index)}") + if nodate := sorted(set(points.id) - set(out.id)): + print(f"no date, so no window fetched: {nodate}") diff --git a/skills/chirts-temperature/SKILL.md b/skills/chirts-temperature/SKILL.md index fc7e7d3..15510de 100644 --- a/skills/chirts-temperature/SKILL.md +++ b/skills/chirts-temperature/SKILL.md @@ -15,9 +15,12 @@ Resolve `scripts/fetch.py` relative to this skill's directory, not the user's wo ```sh uv run scripts/fetch.py points.csv temp.parquet --start 2020-01-01 --end 2025-12-31 +uv run scripts/fetch.py points.csv temp.parquet --date-col planting_date --days 100 ``` -`--start` is required; use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `tmax_c`, `tmin_c`, `cell_lat`, `cell_lon`. Values above 60 °C are masked (a few source pixels hold impossible values). The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. +Give one period for all points with `--start`, or a window per point with `--date-col` and `--days`: each point is fetched from the date in that column (day 0) to `--days` later, and the output gains a `day` column counting from 0. Use the per-point form when points have their own dates, such as planting dates; it reads only the dates each point needs. A point with a blank date gets no rows and is listed. Use the requested period, asking if it is unspecified. `--end` defaults to the latest day actually in the store. Explicit dates outside store coverage fail with the available range; do not silently shorten a requested period. Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point and day: `id`, `date`, `tmax_c`, `tmin_c`, `cell_lat`, `cell_lon`. Values above 60 °C are masked (a few source pixels hold impossible values). The script prints any point whose series is all NaN (missing coordinates, outside coverage or source nodata). Report these IDs and leave their coordinates unchanged. + +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `huggingface.co` and the download host it redirects to (`scripts/check.py` in the repository names it) for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. ## Bandwidth @@ -34,7 +37,7 @@ ds = xr.open_zarr("https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/ - Two variables, `Tmax` and `Tmin`, in °C, stored as int16 in 0.1 °C steps; xarray decodes them and turns `-32768` into NaN. Dims `(time, lat, lon)`. - Latitude is descending. Slice north to south: `lat=slice(north, south)`. - Land only, 60°S to 70°N. Sea and lake cells are NaN every day. -- Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. +- Select dates first, pick all points in one vectorised `sel(method="nearest")`, `.compute()` once, raise dask's thread count. For a different window per point, index time with a (point, day) array of positions in the same vectorised `isel`; dask still fetches each chunk once. `select_daily` in `scripts/fetch.py` shows how. - From R, `terra::rast('ZARR:"path.zarr":/Tmax')` works with GDAL 3.11 or newer, but returns raw integers: multiply by 0.1 and set `NAflag` to -32768. ## Caveats diff --git a/skills/chirts-temperature/scripts/fetch.py b/skills/chirts-temperature/scripts/fetch.py index b0024df..2e3dc6f 100644 --- a/skills/chirts-temperature/scripts/fetch.py +++ b/skills/chirts-temperature/scripts/fetch.py @@ -5,22 +5,35 @@ """Daily CHIRTS-ERA5 Tmax and Tmin for points. One row per point and day, degrees C. uv run fetch.py points.csv out.parquet --start 2020-01-01 [--end 2025-12-31] + uv run fetch.py points.csv out.parquet --date-col planting_date --days 100 -points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store.""" +points.csv needs columns id, lat, lon. Writes CSV if out ends in .csv, else parquet. end defaults to the latest day in the store. +With --date-col each point gets its own window, from its date (day 0) to --days later.""" import argparse + import dask +import numpy as np import pandas as pd import xarray as xr STORE = "https://huggingface.co/datasets/aaguilar90/chirts-era5-daily/resolve/main/chirts-era5-daily.zarr" +HOSTS = ["huggingface.co"] # what a firewall must allow, plus the download host it redirects to (scripts/check.py names it) THREADS = 32 # reads wait on the network, not the CPU TMAX_OK = 60 # a few source pixels hold impossible values; mask above this +def open_store() -> xr.Dataset: + # trust_env makes the HTTP client honour HTTPS_PROXY, which sandboxes and institutional networks route through + return xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"client_kwargs": {"trust_env": True}}) + + def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -29,42 +42,91 @@ def validate_points(points: pd.DataFrame) -> pd.DataFrame: return points -def select_daily(ds, points, start, end, lat, lon): - """Nearest cell per point. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" - first, last = pd.to_datetime(ds.time.values[[0, -1]]) - start, end = pd.Timestamp(start), pd.Timestamp(end or last) - if not first <= start <= end <= last: # else the store silently returns a shorter series - raise ValueError(f"dates must be ordered and within {first.date()} to {last.date()}") - on_grid = points.lat.notna() & points.lon.notna() - for col, name in (("lat", lat), ("lon", lon)): - axis = ds[name].values - half = abs(axis[1] - axis[0]) / 2 - on_grid &= points[col].between(axis.min() - half, axis.max() + half) - pts = points[on_grid] - sel = {name: xr.DataArray(pts[col].values, dims="pt", coords={"pt": pts.index}) for col, name in (("lat", lat), ("lon", lon))} +def periods(points: pd.DataFrame, start, end, days) -> tuple[pd.DataFrame, int | None]: + """Each point's day 0 in `date`, and the days after it: one period for all (start, end), or each point's own date and days.""" + if days is None: + if start is None: + raise ValueError("give start, or days with a date column in points") + days = (pd.Timestamp(end) - pd.Timestamp(start)).days if end else None # None: to the store's last day + return points.assign(date=start), days + if "date" not in points: + raise ValueError("a window per point needs a date column") + points = points[pd.to_datetime(points.date).notna()].reset_index(drop=True) # no date, no window + if points.empty: + raise ValueError("no point has a date") + return points, days + + +def select_daily(ds, points, days, lat, lon): + """Nearest cell per point, from its `date` (day 0) to `days` later, or to the store's last day if days is None. + + One vectorised read, so dask fetches each chunk once and in parallel. Off-grid points stay NaN; nearest alone would snap them to the edge cell.""" + times = ds.indexes["time"].normalize() + day0 = pd.to_datetime(points.date).dt.normalize().values + days = (times[-1] - day0.min()).days if days is None else days + want = day0[:, None] + np.arange(max(days, 0) + 1) * np.timedelta64(1, "D") + ti = times.get_indexer(want.ravel()).reshape(want.shape) + if days < 0 or (ti < 0).any(): # else the store silently returns a shorter series + raise ValueError(f"dates must be ordered and within {times[0].date()} to {times[-1].date()}; " + f"not so for ids {points.id[(ti < 0).any(axis=1)].tolist()}") + idx = {} + for col, name in (("lat", lat), ("lon", lon)): # -1 when off grid or missing; it reads the last cell, then is masked + half = abs(float(ds[name][1] - ds[name][0])) / 2 + 1e-6 # slack: points on a cell edge must not round off grid + near = ds.indexes[name].get_indexer(points[col], method="nearest", tolerance=half) + idx[name] = xr.DataArray(near, dims="pt", coords={"pt": points.id.values}) + one = (day0 == day0[0]).all() # one period for all: a plain slice is far cheaper than a per-point time index + idx["time"] = slice(ti[0, 0], ti[0, -1] + 1) if one else xr.DataArray(ti, dims=("pt", "day")) with dask.config.set(num_workers=THREADS): - sub = ds.sel(time=slice(start, end)).sel(**sel, method="nearest").compute() - return sub.reindex(pt=points.index) + sub = ds.isel(idx).compute() + if one: + sub = sub.assign_coords(day=("time", np.arange(days + 1))).swap_dims(time="day").transpose("pt", "day") + keep = (idx[lat] >= 0) & (idx[lon] >= 0) + return sub.where(keep).assign_coords({n: sub[n].where(keep) for n in (lat, lon)}) -def fetch(points: pd.DataFrame, start: str, end: str | None = None) -> pd.DataFrame: - points = validate_points(points) - ds = xr.open_zarr(STORE, consolidated=True, chunks={}, storage_options={"client_kwargs": {"trust_env": True}}) # honour HTTPS_PROXY, which sandboxes use - sub = select_daily(ds[["Tmax", "Tmin"]], points, start, end, "lat", "lon") +def cli_points(p: argparse.ArgumentParser, a: argparse.Namespace) -> pd.DataFrame: + """One period for all points (--start), or a window per point (--date-col with --days).""" + if bool(a.start) == bool(a.date_col) or (a.date_col and (a.days is None or a.end)): + p.error("give --start [--end] for one period, or --date-col with --days for a window per point") + points = pd.read_csv(a.points, dtype={"id": str}) + if a.date_col and a.date_col != "date": + if a.date_col not in points: + p.error(f"no column {a.date_col} in {a.points}") + points = points.drop(columns="date", errors="ignore").rename(columns={a.date_col: "date"}) + return points + + +def daily(ds: xr.Dataset, points: pd.DataFrame, days: int | None) -> pd.DataFrame: + sub = select_daily(ds[["Tmax", "Tmin"]], points, days, "lat", "lon") sub = sub.where(sub <= TMAX_OK) - df = sub.to_dataframe().reset_index().rename(columns={"Tmax": "tmax_c", "Tmin": "tmin_c", "lat": "cell_lat", "lon": "cell_lon"}) - df["id"] = points.id.values[df.pt] + df = sub.to_dataframe().reset_index().rename(columns={"pt": "id", "Tmax": "tmax_c", "Tmin": "tmin_c", "lat": "cell_lat", "lon": "cell_lon"}) df["date"] = df.time.dt.date - return df[["id", "date", "tmax_c", "tmin_c", "cell_lat", "cell_lon"]].astype({"tmax_c": "float32", "tmin_c": "float32"}) + return df[["id", "date", "day", "tmax_c", "tmin_c", "cell_lat", "cell_lon"]].astype({"tmax_c": "float32", "tmin_c": "float32"}) + + +def fetch(points: pd.DataFrame, start: str | None = None, end: str | None = None, days: int | None = None) -> pd.DataFrame: + """One period for all points (start, end), or with days a window per point from its `date` column.""" + keep_day = days is not None + points, days = periods(validate_points(points), start, end, days) + try: + out = daily(open_store(), points, days) + except OSError as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None + return out if keep_day else out.drop(columns="day") if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("points"), p.add_argument("out"), p.add_argument("--start", required=True), p.add_argument("--end") + p.add_argument("points"), p.add_argument("out"), p.add_argument("--start"), p.add_argument("--end") + p.add_argument("--date-col", help="column with one date per point, such as the planting date") + p.add_argument("--days", type=int, help="days to fetch after each point's date; 100 gives days 0 to 100") a = p.parse_args() - out = fetch(pd.read_csv(a.points, dtype={"id": str}), a.start, a.end) + points = cli_points(p, a) + out = fetch(points, a.start, a.end, a.days) out.to_csv(a.out, index=False) if a.out.endswith(".csv") else out.to_parquet(a.out, index=False, compression="zstd") sea = out[["tmax_c", "tmin_c"]].isna().all(axis=1).groupby(out.id).all() print(f"{out.id.nunique()} points, {out.date.min()} to {out.date.max()}, {len(out):,} rows -> {a.out}") if sea.any(): print(f"all-NaN (missing coordinates, outside coverage or source nodata): {list(sea[sea].index)}") + if nodate := sorted(set(points.id) - set(out.id)): + print(f"no date, so no window fetched: {nodate}") diff --git a/skills/elevation/SKILL.md b/skills/elevation/SKILL.md index 7a73385..d301d04 100644 --- a/skills/elevation/SKILL.md +++ b/skills/elevation/SKILL.md @@ -19,6 +19,8 @@ uv run scripts/fetch.py points.csv elevation.parquet Needs only `uv` ([install](https://docs.astral.sh/uv/getting-started/installation/)); it fetches Python and the packages on first run. Output is one row per point: `id`, `elevation_m`. The script prints IDs left NaN because coordinates are missing, a tile is absent, or the source cell is nodata. Network and other read errors fail the run. `(0, 0)` is a valid coordinate, not an automatic missing-value sentinel. +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `copernicus-dem-90m.s3.amazonaws.com` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. + ## If you need to read the tiles yourself Tiles are named by their south-west corner, so (-1.95, 30.06) is in `S02_00_E030_00`: diff --git a/skills/elevation/scripts/fetch.py b/skills/elevation/scripts/fetch.py index 2e42e31..947d19c 100644 --- a/skills/elevation/scripts/fetch.py +++ b/skills/elevation/scripts/fetch.py @@ -22,6 +22,7 @@ "https://copernicus-dem-90m.s3.amazonaws.com/" "Copernicus_DSM_COG_30_{t}_DEM/Copernicus_DSM_COG_30_{t}_DEM.tif" ) +HOSTS = ["copernicus-dem-90m.s3.amazonaws.com"] # what a firewall must allow def tile_url(lat: float, lon: float) -> str: @@ -33,7 +34,10 @@ def tile_url(lat: float, lon: float) -> str: def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -58,7 +62,7 @@ def fetch(points: pd.DataFrame) -> pd.DataFrame: except RasterioIOError as exc: # A missing ocean tile is expected; a failed connection is not nodata. if "404" not in str(exc): - raise + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return pd.DataFrame({"id": points.id, "elevation_m": out.astype("float32")}) diff --git a/skills/soilgrids-soil/SKILL.md b/skills/soilgrids-soil/SKILL.md index f4c1b2d..1d400a4 100644 --- a/skills/soilgrids-soil/SKILL.md +++ b/skills/soilgrids-soil/SKILL.md @@ -25,6 +25,8 @@ Expect about 20 to 30 s for 1,000 points and the default 27 layers. Each layer c Use `--res 1000` when point locations are uncertain by more than a few hundred metres, such as tricot sites, which are displaced up to 1.5 km for privacy. A 1 km average then describes the area the farm is in better than one 250 m cell. It has the mean only, so prediction intervals stay at 250 m. +The script needs `pypi.org` and `files.pythonhosted.org` on first run and `files.isric.org` for the data. If it reports a host it cannot reach, the network is blocked, not the data missing: say so, name the host, and give the user the command to run on an open network. Do not switch source or estimate values. + ## Properties and units Values are converted from SoilGrids' stored integers to the units below. diff --git a/skills/soilgrids-soil/scripts/fetch.py b/skills/soilgrids-soil/scripts/fetch.py index 3b5093d..c04234b 100644 --- a/skills/soilgrids-soil/scripts/fetch.py +++ b/skills/soilgrids-soil/scripts/fetch.py @@ -29,6 +29,7 @@ DEFAULT_PROPS = "clay,sand,silt,soc,nitrogen,phh2o,cec,bdod,cfvo" DEFAULT_DEPTHS = "0-5cm,5-15cm,15-30cm" GDAL = dict(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", GDAL_HTTP_MAX_RETRY=3, GDAL_HTTP_RETRY_DELAY=2) +HOSTS = ["files.isric.org"] # what a firewall must allow def layer(prop: str, depth: str, stat: str, res: int, lat: np.ndarray, lon: np.ndarray) -> np.ndarray: @@ -45,7 +46,10 @@ def layer(prop: str, depth: str, stat: str, res: int, lat: np.ndarray, lon: np.n def validate_points(points: pd.DataFrame) -> pd.DataFrame: """Unique ids so outputs join cleanly; WGS84 ranges catch swapped or projected coordinates.""" + if missing := {"id", "lat", "lon"} - set(points.columns): + raise ValueError(f"points need columns id, lat, lon; missing {sorted(missing)}") points = points.reset_index(drop=True) + points = points.assign(lat=pd.to_numeric(points.lat), lon=pd.to_numeric(points.lon)) if points.id.isna().any() or points.id.duplicated().any(): raise ValueError("points need unique, non-empty ids") for col, limit in (("lat", 90), ("lon", 180)): @@ -64,9 +68,12 @@ def fetch(points: pd.DataFrame, props: list[str], depths: list[str], stat: str = jobs = [(p, "0-30cm" if p == "ocs" else d) for p in props for d in depths] jobs = list(dict.fromkeys(jobs)) lat, lon = points.lat.to_numpy(float), points.lon.to_numpy(float) - with ThreadPoolExecutor(min(32, len(jobs))) as ex: # each layer opens its own 6 MB VRT; network-bound - cols = ex.map(lambda j: layer(*j, stat, res, lat, lon), jobs) - out = {f"{p}_{d.replace('-', '_')}": c for (p, d), c in zip(jobs, cols)} + try: + with ThreadPoolExecutor(min(32, len(jobs))) as ex: # each layer opens its own 6 MB VRT; network-bound + cols = ex.map(lambda j: layer(*j, stat, res, lat, lon), jobs) + out = {f"{p}_{d.replace('-', '_')}": c for (p, d), c in zip(jobs, cols)} + except OSError as exc: # a block, or a 404 if the store moved; check.py tells which + raise ConnectionError(f"could not read {', '.join(HOSTS)} ({exc}). If the network blocks it, run scripts/check.py; see 'Network' in the repository SKILL.md") from None return pd.DataFrame({"id": points.id, **out}) diff --git a/tests/points.csv b/tests/points.csv new file mode 100644 index 0000000..d294e5d --- /dev/null +++ b/tests/points.csv @@ -0,0 +1,3 @@ +id,lat,lon,planting_date +tz-selian,-3.36,36.63,2023-11-01 +bj-niaouli,6.74,2.14,2024-04-15 diff --git a/tests/smoke.py b/tests/smoke.py new file mode 100644 index 0000000..1bf534b --- /dev/null +++ b/tests/smoke.py @@ -0,0 +1,48 @@ +"""Fetch two points from every live store and check the answers. Needs uv and open internet. + + python3 tests/smoke.py + +Catches a store that moved, changed layout or stopped answering. AgERA5 runs only when CDSAPI_KEY is set.""" + +import csv +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).parents[1] +WINDOW = ["--date-col", "planting_date", "--days", "5"] # 2 points x days 0 to 5 +# (label, skill, arguments, columns that must hold values, rows expected) +JOBS = [ + ("chirps-rainfall", "chirps-rainfall", WINDOW, ["precip_mm"], 12), + ("chirps-rainfall one period", "chirps-rainfall", ["--start", "2024-01-01", "--end", "2024-01-03"], ["precip_mm"], 6), + ("chirts-temperature", "chirts-temperature", WINDOW, ["tmax_c", "tmin_c"], 12), + ("agera5-climate", "agera5-climate", WINDOW, ["et0", "srad"], 12), + ("elevation", "elevation", [], ["elevation_m"], 2), + ("soilgrids-soil", "soilgrids-soil", ["--props", "clay", "--depths", "0-5cm", "--res", "1000"], ["clay_0_5cm"], 2), +] + + +def fetch(skill: str, args: list[str]) -> list[dict]: + out = Path(tempfile.mkdtemp()) / f"{skill}.csv" + script = ROOT / "skills" / skill / "scripts" / "fetch.py" + subprocess.run(["uv", "run", str(script), str(ROOT / "tests" / "points.csv"), str(out), *args], check=True) + return list(csv.DictReader(out.open())) + + +if __name__ == "__main__": + failed = [] + for label, skill, args, columns, n in JOBS: + if skill == "agera5-climate" and not os.environ.get("CDSAPI_KEY"): + print(f"skipped {label}: no CDSAPI_KEY") + continue + try: + rows = fetch(skill, args) + assert len(rows) == n, f"{len(rows)} rows, expected {n}" + assert all(r[c] != "" for r in rows for c in columns), f"missing values in {columns}" + print(f"ok {label}: {rows[0]}") + except (AssertionError, subprocess.CalledProcessError) as exc: + failed.append(label) + print(f"FAILED {label}: {exc}") + sys.exit(f"failed: {failed}" if failed else None) diff --git a/tests/test_fetch.py b/tests/test_fetch.py new file mode 100644 index 0000000..44e879a --- /dev/null +++ b/tests/test_fetch.py @@ -0,0 +1,142 @@ +# /// script +# requires-python = ">=3.12" +# dependencies = ["pytest", "xarray", "zarr>=3", "icechunk", "dask", "fsspec", "aiohttp", "pandas", "pyarrow", "rasterio", "numpy"] +# /// +"""Offline tests on synthetic stores. No network. The dependencies are every skill's, plus pytest. + + uv run tests/test_fetch.py +""" + +import argparse +import importlib.util +import inspect +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import xarray as xr + +ROOT = Path(__file__).parents[1] +WEATHER = ["chirps-rainfall", "chirts-temperature", "agera5-climate"] +SKILLS = [*WEATHER, "elevation", "soilgrids-soil"] + + +def load(path: Path): + spec = importlib.util.spec_from_file_location(path.parent.parent.name.replace("-", "_") + "_" + path.stem, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +MODS = {s: load(ROOT / "skills" / s / "scripts" / "fetch.py") for s in SKILLS} +# per skill: latitude and longitude names, store variables, north-to-south grid, value column, output value on day 10 of the year +STORES = { + "chirps-rainfall": ("latitude", "longitude", ["precipitation_surface"], True, "precip_mm", 86400.0), + "chirts-temperature": ("lat", "lon", ["Tmax", "Tmin"], True, "tmax_c", 1.0), + "agera5-climate": ("latitude", "longitude", [v[0] for v in MODS["agera5-climate"].VARS.values()], False, "et0", 1.0), +} +POINTS = pd.DataFrame({ + "id": ["a", "b", "c", "d"], + "lat": [5.1, -3.3, 5.2, np.nan], # d has no coordinates + "lon": [2.2, 36.7, 2.3, 2.0], + "date": ["2023-02-01", "2023-06-15", None, "2023-03-01"], # c has no date +}) + + +def store(lat: str, lon: str, variables: list[str], descending: bool) -> xr.Dataset: + """A year on a 0.5 degree grid over Africa. Every cell holds a tenth of the day of the year, so a window is easy to check.""" + time = pd.date_range("2023-01-01", "2023-12-31") + lats = np.arange(-9.75, 10, 0.5)[:: -1 if descending else 1] + lons = np.arange(-9.75, 45, 0.5) + doy = np.broadcast_to(np.arange(1, 366, dtype="float32")[:, None, None] / 10, (365, len(lats), len(lons))) + return xr.Dataset({v: (("time", lat, lon), doy.copy()) for v in variables}, coords={"time": time, lat: lats, lon: lons}) + + +@pytest.fixture(params=WEATHER) +def weather(request, monkeypatch): + mod, (lat, lon, variables, descending, column, scale) = MODS[request.param], STORES[request.param] + monkeypatch.setattr(mod, "open_store", lambda: store(lat, lon, variables, descending)) + return mod, column, scale + + +def test_shared_helpers_are_identical(): + """Each skill is self-contained, so the helpers are copied. This catches a copy that drifts.""" + shared = {"validate_points": SKILLS, "select_daily": WEATHER, "periods": WEATHER, "cli_points": WEATHER} + for name, skills in shared.items(): + assert len({inspect.getsource(getattr(MODS[s], name)) for s in skills}) == 1, name + + +def test_check_knows_every_host(): + check = load(ROOT / "scripts" / "check.py") + assert {s: check.NEEDS[s] for s in SKILLS} == {s: MODS[s].HOSTS for s in SKILLS} + + +def test_window_per_point(weather): + mod, column, scale = weather + out = mod.fetch(POINTS, days=10) + assert list(out.columns[:3]) == ["id", "date", "day"] + assert set(out.id) == {"a", "b", "d"} # c has no date, so no window + assert out.groupby("id").size().eq(11).all() + a = out[out.id == "a"] + assert list(a.day) == list(range(11)) and str(a.date.iloc[0]) == "2023-02-01" + np.testing.assert_allclose(a[column], np.arange(32, 43) / 10 * scale, rtol=1e-5) # 1 February is day 32 + assert a.cell_lat.iloc[0] == pytest.approx(5.25) and a.cell_lon.iloc[0] == pytest.approx(2.25) + assert out[out.id == "b"][column].iloc[0] == pytest.approx(16.6 * scale, rel=1e-5) # 15 June + assert out[out.id == "d"][column].isna().all() # no coordinates: rows stay, values are NaN + + +def test_one_period_for_all_points(weather): + mod, column, scale = weather + out = mod.fetch(POINTS, "2023-03-01", "2023-03-05") + assert "day" not in out and len(out) == 5 * len(POINTS) + assert out[out.id == "a"][column].iloc[0] == pytest.approx(6.0 * scale, rel=1e-5) + + +def test_point_on_a_cell_edge_is_on_grid(weather): + mod, column, _ = weather + edge = pd.DataFrame({"id": ["e"], "lat": [5.0 + 1e-9], "lon": [2.0]}) # float noise past the edge between two cells + assert mod.fetch(edge, "2023-03-01", "2023-03-02")[column].notna().all() + + +def test_a_failed_read_points_to_check(weather, monkeypatch): + mod, _, _ = weather + monkeypatch.setattr(mod, "open_store", lambda: (_ for _ in ()).throw(OSError("Cannot connect to host 127.0.0.1:9"))) + with pytest.raises(ConnectionError, match="check.py"): + mod.fetch(POINTS, "2023-03-01", "2023-03-02") + + +def test_window_outside_the_store_fails(weather): + mod, _, _ = weather + late = POINTS.assign(date="2023-12-28") + with pytest.raises(ValueError, match="within"): # never shorten a requested period silently + mod.fetch(late, days=10) + with pytest.raises(ValueError, match="give start"): + mod.fetch(POINTS) + + +@pytest.mark.parametrize("skill", SKILLS) +def test_points_are_validated(skill): + validate = MODS[skill].validate_points + with pytest.raises(ValueError, match="missing"): + validate(pd.DataFrame({"id": ["a"], "latitude": [1.0], "longitude": [2.0]})) + with pytest.raises(ValueError, match="unique"): + validate(pd.DataFrame({"id": ["a", "a"], "lat": [1.0, 1.0], "lon": [2.0, 2.0]})) + with pytest.raises(ValueError, match="outside"): + validate(pd.DataFrame({"id": ["a"], "lat": [36.7], "lon": [-183.3]})) + + +def test_cli_takes_one_mode(tmp_path): + mod = MODS["chirps-rainfall"] + POINTS.rename(columns={"date": "planting_date"}).to_csv(tmp_path / "points.csv", index=False) + parser = argparse.ArgumentParser() + args = dict(points=tmp_path / "points.csv", start=None, end=None, date_col="planting_date", days=10) + assert "date" in mod.cli_points(parser, argparse.Namespace(**args)) + for wrong in ({"days": None}, {"start": "2023-01-01"}, {"date_col": None}, {"date_col": "sowing"}): + with pytest.raises(SystemExit): + mod.cli_points(parser, argparse.Namespace(**(args | wrong))) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, *sys.argv[1:]]))