diff --git a/.github/workflows/skills.yml b/.github/workflows/skills.yml new file mode 100644 index 00000000..bfcfeb03 --- /dev/null +++ b/.github/workflows/skills.yml @@ -0,0 +1,316 @@ +name: skills +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'skills.sh.json' + - 'pyproject.toml' + - 'src/sedpack/.agents/skills/**' + - 'tests/io/test_skills.py' + - '.github/workflows/skills.yml' + merge_group: # Needed for required workflows + # Run after a review has been submitted (this is a required workflow which + # might not be triggered when no code changes -- trigger before going to + # merge queue). + pull_request_review: + types: [submitted] + +jobs: + validate_skills: + name: Validate skills + runs-on: ubuntu-22.04 + steps: + - name: Checkout the code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Set up Python 3.11 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.11' + cache: 'pip' + - name: Install schema and skill validators + run: | + python -m pip install --upgrade pip + pip install --upgrade check-jsonschema skills-ref pytest + - name: Validate skills.sh.json against schema + run: | + check-jsonschema --schemafile https://skills.sh/schemas/skills.sh.schema.json skills.sh.json + - name: Validate configured skill paths and SKILL.md files + run: | + python - <<'EOF' + import json + from pathlib import Path + from skills_ref import validate + + config = json.loads(Path("skills.sh.json").read_text(encoding="utf-8")) + expected_schema = "https://skills.sh/schemas/skills.sh.schema.json" + assert config.get("$schema") == expected_schema, ( + f"Expected $schema={expected_schema!r}, got {config.get('$schema')!r}" + ) + paths = config.get("paths", []) + assert paths, "skills.sh.json must define non-empty 'paths'" + + for rel_path in paths: + container = Path(rel_path) + assert container.is_dir(), ( + f"Configured skills path does not exist: {rel_path}" + ) + skill_dirs = [p.parent for p in sorted(container.rglob("SKILL.md"))] + assert skill_dirs, f"No SKILL.md found under {rel_path}" + for skill_dir in skill_dirs: + errors = validate(skill_dir) + assert not errors, f"Validation failed for {skill_dir}: {errors}" + print(f"Validated skill: {skill_dir}") + EOF + - name: Run unit tests for bundled skill metadata + run: | + PYTHONPATH=./src pytest tests/io/test_skills.py + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: "24" + - name: Verify npx skills discovery and installation via skills.sh.json + run: | + npx --yes skills add . --list + npx --yes skills add . --skill sedpack --agent gemini-cli -y + test -f .agents/skills/sedpack/SKILL.md + + env_managers: + name: Test environment managers + runs-on: ubuntu-22.04 + steps: + - name: Checkout the code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Set up Python 3.11 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.11' + cache: 'pip' + - name: Setup Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f + - name: Build sedpack wheel and install environment managers + library-skills + run: | + python -m pip install --upgrade pip + pip install "maturin>=1.7,<2.0" library-skills virtualenv uv poetry pdm pipenv hatch + pip wheel --no-deps --no-build-isolation . -w "$RUNNER_TEMP/dist" + python - "$RUNNER_TEMP/dist" <<'EOF' + import sys, zipfile + from pathlib import Path + wheels = list(Path(sys.argv[1]).glob("sedpack-*.whl")) + assert len(wheels) == 1, wheels + with zipfile.ZipFile(wheels[0]) as zf: + names = zf.namelist() + assert "sedpack/.agents/skills/sedpack/SKILL.md" in names, names + rec = [n for n in names if n.endswith("RECORD")][0] + rec_text = zf.read(rec).decode("utf-8") + assert "sedpack/.agents/skills/sedpack/SKILL.md" in rec_text + EOF + - name: Test standard venv + library-skills (wheel and editable installs) + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/venv_test" + mkdir -p "$WORK/proj" "$WORK/edit_proj" + + cd "$WORK/proj" + python -m venv .venv + .venv/bin/pip install --no-deps "$WHEEL" + library-skills install --skill sedpack --yes --claude + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + test -L .claude/skills/sedpack + test -f .claude/skills/sedpack/SKILL.md + python -c 'import os; link = os.readlink(".agents/skills/sedpack"); assert not os.path.isabs(link), link' + library-skills --check --claude + + # Also test --copy mode + rm -rf .agents .claude + library-skills install --skill sedpack --yes --copy + test ! -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + + # Also test editable install discovery via direct_url.json + cd "$WORK/edit_proj" + python -m venv .venv + .venv/bin/pip install "maturin>=1.7,<2.0" + .venv/bin/pip install --no-deps --no-build-isolation -e "$GITHUB_WORKSPACE" + library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + library-skills --check + - name: Test virtualenv + library-skills + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/virtualenv_test" + mkdir -p "$WORK/proj" + cd "$WORK/proj" + virtualenv .venv + .venv/bin/pip install --no-deps "$WHEEL" + library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + python -c 'import os; link = os.readlink(".agents/skills/sedpack"); assert not os.path.isabs(link), link' + library-skills --check + - name: Test uv + library-skills + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/uv_test" + mkdir -p "$WORK/proj" + cd "$WORK/proj" + uv venv .venv + uv pip install --python .venv/bin/python --no-deps "$WHEEL" + library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + python -c 'import os; link = os.readlink(".agents/skills/sedpack"); assert not os.path.isabs(link), link' + library-skills --check + - name: Test poetry (in-project and external cache) + library-skills + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/poetry_test" + mkdir -p "$WORK/inproj" "$WORK/cache_proj" "$WORK/home" + export HOME="$WORK/home" + + # 1. Poetry with virtualenvs.in-project = true + cd "$WORK/inproj" + cat > pyproject.toml <<'EOF' + [project] + name = "demo-poetry-inproj" + version = "0.1.0" + requires-python = ">=3.11" + dependencies = ["sedpack"] + EOF + poetry config virtualenvs.in-project true --local + poetry env use python + poetry run pip install --no-deps "$WHEEL" + library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + library-skills --check + + # 2. Poetry with default external cache virtualenv + cd "$WORK/cache_proj" + cat > pyproject.toml <<'EOF' + [project] + name = "demo-poetry-cache" + version = "0.1.0" + requires-python = ">=3.11" + dependencies = ["sedpack"] + EOF + poetry config virtualenvs.in-project false --local + poetry env use python + poetry run pip install --no-deps "$WHEEL" + UV_PROJECT_ENVIRONMENT="$(poetry env info --path)" library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + UV_PROJECT_ENVIRONMENT="$(poetry env info --path)" library-skills --check + - name: Test pdm + library-skills + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/pdm_test" + mkdir -p "$WORK/proj" "$WORK/home" + export HOME="$WORK/home" + cd "$WORK/proj" + cat > pyproject.toml <<'EOF' + [project] + name = "demo-pdm" + version = "0.1.0" + requires-python = ">=3.11" + dependencies = ["sedpack"] + EOF + pdm venv create --force python + PDM_PY=$(pdm info --python) + uv pip install --python "$PDM_PY" --no-deps "$WHEEL" + library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + library-skills --check + - name: Test pipenv (in-project and external virtualenv) + library-skills + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/pipenv_test" + mkdir -p "$WORK/inproj" "$WORK/ext_proj" "$WORK/home" + export HOME="$WORK/home" + + # 1. Pipenv with PIPENV_VENV_IN_PROJECT=1 + cd "$WORK/inproj" + export PIPENV_VENV_IN_PROJECT=1 + pipenv --python "$(which python)" + PIPENV_PY=$(pipenv --py) + uv pip install --python "$PIPENV_PY" --no-deps "$WHEEL" + library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + library-skills --check + unset PIPENV_VENV_IN_PROJECT + + # 2. Pipenv default external virtualenv + cd "$WORK/ext_proj" + export WORKON_HOME="$WORK/home/.virtualenvs" + pipenv --python "$(which python)" + PIPENV_PY=$(pipenv --py) + uv pip install --python "$PIPENV_PY" --no-deps "$WHEEL" + UV_PROJECT_ENVIRONMENT="$(pipenv --venv)" library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + UV_PROJECT_ENVIRONMENT="$(pipenv --venv)" library-skills --check + - name: Test hatch + library-skills + run: | + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/hatch_test" + mkdir -p "$WORK/proj" "$WORK/home" + export HOME="$WORK/home" + cd "$WORK/proj" + cat > pyproject.toml <<'EOF' + [project] + name = "demo-hatch" + version = "0.1.0" + requires-python = ">=3.11" + dependencies = ["sedpack"] + EOF + hatch env create + HATCH_ENV_DIR=$(hatch env find) + uv pip install --python "$HATCH_ENV_DIR/bin/python" --no-deps "$WHEEL" + UV_PROJECT_ENVIRONMENT="$HATCH_ENV_DIR" library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + UV_PROJECT_ENVIRONMENT="$HATCH_ENV_DIR" library-skills --check + - name: Test conda (prefix and named environments) + library-skills + run: | + if ! command -v conda >/dev/null 2>&1; then + if [ -x "$CONDA/bin/conda" ]; then + export PATH="$CONDA/bin:$PATH" + else + echo "conda not installed on runner; skipping conda step" + exit 0 + fi + fi + WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl) + WORK="$RUNNER_TEMP/conda_test" + mkdir -p "$WORK/prefix_proj" "$WORK/named_proj" + + # 1. Local prefix environment (./.conda) + cd "$WORK/prefix_proj" + conda create --yes --quiet --prefix ./.conda python=3.11 pip + ./.conda/bin/pip install --no-deps "$WHEEL" + CONDA_PREFIX="$WORK/prefix_proj/.conda" library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + CONDA_PREFIX="$WORK/prefix_proj/.conda" library-skills --check + + # 2. Named conda environment + cd "$WORK/named_proj" + conda create --yes --quiet -n sedpack-conda-env python=3.11 pip + conda run -n sedpack-conda-env pip install --no-deps "$WHEEL" + CONDA_ENV_PATH=$(conda info --envs | awk '/^sedpack-conda-env / {print $NF}') + CONDA_PREFIX="$CONDA_ENV_PATH" library-skills install --skill sedpack --yes + test -L .agents/skills/sedpack + test -f .agents/skills/sedpack/SKILL.md + CONDA_PREFIX="$CONDA_ENV_PATH" library-skills --check diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..52b8512f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,61 @@ +# AGENTS.md + +Sedpack is a scalable and efficient sharded dataset storage and streaming +library: a Python package (`src/sedpack/`), a Rust native extension (`rust/`, +built with PyO3 and `maturin` as `sedpack._sedpack_rs`), a user-facing LLM +agent skill (`src/sedpack/.agents/skills/sedpack/SKILL.md`), tutorials +(`docs/tutorials/`), a `pytest` suite (`tests/`), and an Astro Starlight +documentation website (`website/`). + +If you are using `sedpack` in another project rather than developing `sedpack` +itself, read `src/sedpack/.agents/skills/sedpack/SKILL.md` for the high-level +`sedpack.io` API guide. + +## Before Opening a Pull Request + +- Read the contribution guidelines in `docs/contributing.md`. +- Keep each pull request focused on a single problem with a concise + description of what changed, why, and how it was tested. Avoid unrelated + refactoring, renames, formatting churn, version bumps, or `requirements.txt` + lock file churn unless explicitly requested. +- Every tracked `.py` and `.sh` file (except generated FlatBuffers files under + `src/sedpack/io/flatbuffer/shardfile/`) must include the Apache 2.0 + copyright header. Verify with `./tools/check_copyright.sh`. + +## Working in the Repository + +- **Build and editable install**: Activate the virtual environment, install + dependencies (`pip install --editable ".[dev,tf]"`), and rebuild the Rust + extension after any change in `rust/` via `maturin develop --release`. +- **Unit tests**: Run `python -m pytest` from the repository root. When + TensorFlow is installed (`".[dev,tf]"`), no tests should be skipped. +- **Python formatting (`yapf`)**: Configured in `.style.yapf`. Check + formatting with `yapf --recursive --diff .` and format in-place with + `yapf --recursive --in-place .`. +- **Python type checking (`mypy`)**: Configured in `mypy.ini` (strict mode for + `sedpack.*`). Run `PYTHONPATH=./src mypy --no-color-output src docs`. +- **Python linting (`pylint`)**: Configured in `.pylintrc`. Run + `./tools/run_pylint.sh` (which runs `pylint *.py src docs`). +- **Rust checks**: From `rust/`, run `cargo +nightly fmt -- --check`, + `cargo +nightly clippy --all-targets`, and `cargo test`. +- **Markdown linting (`markdownlint`)**: All `*.md` and `**/*.md` files are + checked against `.markdownlint.json`: + - Wrap prose, list items, and block quotes at 80 characters (120 for + headings; tables and fenced code blocks are exempt). + - Use a hyphen followed by 3 spaces for unordered list items and a number + plus period followed by 2 spaces for ordered list items, with 4-space + indentation for nested lists and continuation lines. + - Always specify a language tag on fenced code blocks (`python`, `bash`, + `text`, `json`, `toml`). +- **Spell checking (`cspell`)**: Checked in CI against `cspell.json` and + `project-words.txt`. If a change introduces legitimate domain terms or + identifiers not recognized by `cspell`, add them in sorted order to + `project-words.txt`. +- **Generated files and skill synchronization**: + - Never hand-edit generated FlatBuffers bindings + (`src/sedpack/io/flatbuffer/shardfile/*.py` or + `rust/src/shard_generated.rs`); regenerate them from + `src/sedpack/io/flatbuffer/shard.fbs`. + - When modifying the public `sedpack.io` API, update + `src/sedpack/.agents/skills/sedpack/SKILL.md` so the bundled agent skill + stays accurate. diff --git a/README.md b/README.md index a2d4bd7c..8fdf7867 100644 --- a/README.md +++ b/README.md @@ -13,58 +13,206 @@ See the documentation website: ## Install -### Dependencies +### Package install + +```bash +pip install sedpack +``` [TensorFlow 2.x](https://www.tensorflow.org/install) is an optional dependency -required only when using the `"tfrec"` shard format or `Dataset.as_tfdataset`. -It can be installed via `pip install "sedpack[tf]"`. +required only when using the `"tfrec"` shard format or `Dataset.as_tfdataset`: -Development dependencies: +```bash +pip install "sedpack[tf]" +``` -- python-dev and gcc for [xxhash](https://pypi.org/project/xxhash/) +### Development install -### Dataset install +Development prerequisites: -#### Development install +- `python3-dev` and `gcc` for [xxhash](https://pypi.org/project/xxhash/) +- [Install Rust](https://www.rust-lang.org/tools/install) (required by + `maturin` to build `sedpack._sedpack_rs`) -1. Clone the repository: `git clone https://github.com/google/sedpack` -2. Install dependencies: `python3 -m pip install --require-hashes -r requirements.txt` -3. Install the package in development mode: - `python3 -m pip install --editable ".[dev]"` (or - `python3 -m pip install --editable .` for a minimal installation without - TensorFlow and development tools) +Steps: -#### Rust install +1. Clone the repository and create a virtual environment: -- Activate your Python virtual environment -- [Install Rust](https://www.rust-lang.org/tools/install) -- Run `maturin develop --release` -- Run `python -m pytest` from the project root directory -- no tests should - be skipped when TensorFlow is installed (e.g., via - `pip install -e ".[dev]"`) + ```bash + git clone https://github.com/google/sedpack + cd sedpack + python3 -m venv .venv + source .venv/bin/activate + ``` -### Update dependencies +2. Install pinned dependencies (optional): + + ```bash + python3 -m pip install --require-hashes -r requirements.txt + ``` + +3. Install the package in editable mode: + + ```bash + python3 -m pip install --editable ".[dev,tf]" + ``` -Make sure to have: `sudo apt install python3 python3-pip python3-venv` and -activated the virtual environment. + Or for a minimal installation without TensorFlow and development tools: -Install requirements: `pip install --require-hashes -r base-tooling-requirements.txt` + ```bash + python3 -m pip install --editable . + ``` -Update: `pip-compile pyproject.toml --generate-hashes --upgrade` and commit requirements.txt. +4. Rebuild the Rust extension after changes in `rust/` and run unit tests: + + ```bash + maturin develop --release + python -m pytest + ``` + + No tests should be skipped when TensorFlow is installed via + `pip install -e ".[dev,tf]"`. + +### Update dependencies -#### Package install +With an active virtual environment, regenerate `requirements.txt` and commit the +updated lock file: -`pip install sedpack` (or `pip install "sedpack[tf]"` to include optional -TensorFlow support) +```bash +uv pip compile pyproject.toml --generate-hashes -o requirements.txt +``` ### Tutorial -A tutorial and documentation is available at +Tutorials and documentation are available at [https://google.github.io/sedpack/](https://google.github.io/sedpack/). Code for the tutorials is available in the `docs/tutorials` directory. For a -"hello world" see -[https://google.github.io/sedpack/tutorials/mnist/](https://google.github.io/sedpack/tutorials/mnist/). +"hello world" example, see the +[MNIST tutorial](https://google.github.io/sedpack/tutorials/mnist/). + +## AI Agents and LLM Skills + +Sedpack includes built-in documentation for AI coding assistants and LLM agent +frameworks: + +- **Using `sedpack` in your project (`SKILL.md`)**: The high-level + `sedpack.io` API skill is bundled inside the package at + `src/sedpack/.agents/skills/sedpack/SKILL.md` (and declared in + `skills.sh.json` at the repository root). + - **Option 1 — Link from your Python environment via + [Library Skills](https://library-skills.io/) (recommended for `.venv` + projects)**: Scans your project's installed Python packages and creates + a relative directory symlink in `.agents/skills/sedpack` so the skill + always matches your installed `sedpack` version and updates + automatically on `pip install -U sedpack`: + + ```bash + # Interactive discovery and installation: + uvx library-skills + + # Non-interactive install into .agents/skills/sedpack: + uvx library-skills install --skill sedpack --yes + + # Also link into .claude/skills/sedpack, or copy files instead of symlinking: + uvx library-skills install --skill sedpack --yes --claude + uvx library-skills install --skill sedpack --yes --copy + + # Inspect status, check symlinks in CI, or repair after recreating .venv: + uvx library-skills list + uvx library-skills --check + uvx library-skills --yes + ``` + + `library-skills` automatically detects `.venv` (`venv`, `virtualenv`, + `uv`, `pdm`, and in-project `poetry`/`pipenv`) and Conda + (`CONDA_PREFIX`). For external virtual environments (such as Poetry's + default cache, Pipenv, or Hatch), pass `UV_PROJECT_ENVIRONMENT`: + + ```bash + UV_PROJECT_ENVIRONMENT="$(poetry env info --path)" uvx library-skills install --skill sedpack --yes + ``` + + - **Option 2 — Install from GitHub via + [Agent Skills CLI (`npx skills`)](https://skills.sh/)**: Installs the + skill directly from `google/sedpack` (or a local checkout) into + project-local or user-global (`-g`) directories across 40+ coding + agents: + + ```bash + # Interactive project-local install from GitHub: + npx skills add google/sedpack + + # Non-interactive install (optionally targeting a specific agent): + npx skills add google/sedpack --skill sedpack -y + npx skills add google/sedpack --skill sedpack --agent gemini-cli -y + npx skills add google/sedpack --skill sedpack --agent claude-code -y + + # Global install into ~/.agents/skills/sedpack: + npx skills add google/sedpack --skill sedpack -g -y + + # List available skills in the repo, or check/update installed skills: + npx skills add google/sedpack --list + npx skills check + npx skills update + ``` + + - **Option 3 — Link with Gemini CLI**: Link the bundled skill directory + directly from your active Python environment (into `~/.gemini/skills` or + project-local `.gemini/skills` with `--scope workspace`): + + ```bash + gemini skills link "$(python -c 'import importlib.resources as r; print(r.files("sedpack") / ".agents/skills/sedpack")')" + ``` + + - **Updating skills when `sedpack` updates (automatic vs. manual, local + `.venv` vs. global `~/`)**: + - **Project-local `.venv` symlink (`uvx library-skills`) — + Automatic**: Because `.agents/skills/sedpack` is a directory symlink + into `site-packages/sedpack/.agents/skills/sedpack`, upgrading + `sedpack` in-place (`pip install -U sedpack`, + `uv sync --upgrade-package sedpack`, or `poetry update sedpack`) + updates `SKILL.md` immediately with no extra command, and keeps each + project isolated to its own `.venv` version. + - **Recreating `.venv` or upgrading Python minor versions (`3.12` -> + `3.13`)**: If `.venv` is recreated with the *same* Python minor + version, the relative symlink resolves again as soon as `sedpack` is + reinstalled. If you upgrade Python minor versions (which changes + `lib/python3.X/site-packages`), repair the symlink with: + + ```bash + uvx library-skills --yes + ``` + + - **Project-local file copy (`uvx library-skills --copy`) — Manual**: + Copied files are static snapshots and do not update on + `pip install -U sedpack`. Re-sync after upgrading `sedpack`: + + ```bash + rm -rf .agents/skills/sedpack + uvx library-skills install --skill sedpack --yes --copy + ``` + + - **GitHub-installed skills (`npx skills`, local or global `-g`) — + Manual**: `npx skills` tracks the `google/sedpack` GitHub repository + rather than your `.venv` (and `-g` shares a single global version + across all projects). Check for and pull updates from GitHub with: + + ```bash + npx skills check + npx skills update + ``` + + - **Gemini CLI link (`gemini skills link`)**: Updates automatically on + in-place `pip install -U sedpack` in the linked environment (run + `/skills reload` in an active Gemini CLI session), or on `git pull` + for an editable install (`pip install -e .`). Re-run + `gemini skills link` if the linked `.venv` is moved or upgraded to a + new Python minor version. + +- **Contributing to `sedpack` (`AGENTS.md`)**: Repository architecture, build + commands, test suites, and linter rules for coding agents working on + `sedpack` itself are documented in `AGENTS.md` at the repository root. ## Disclaimer diff --git a/project-words.txt b/project-words.txt index 9b10ecbf..f3203923 100644 --- a/project-words.txt +++ b/project-words.txt @@ -14,6 +14,7 @@ Invernizzi Josyula Karel Král +LLM Luca MNIST Mangard @@ -36,16 +37,20 @@ Yokeable arange argmax argnames +asarray asyncstdlib bldr booktitle burszteindc byteswap cdylib +clippy codegen compresslevel +conda convnet crossentropy +cspell ddof deepcopy deque @@ -55,6 +60,7 @@ dtypes dunder einsum fbapi +filesystems flatbuffer flate frombuffer @@ -70,21 +76,27 @@ itemsize kwarguments linalg logpdf +markdownlint mathbb mathjax namedtuple ndarray newbyteorder +npx parseable patchelf +pdm perfcounters perfetto pickleable +pipenv +pipx popitem popleft pyarray pyclass pyfunction +pylintrc pymethods pymodule retryable @@ -95,9 +107,13 @@ satteri savez sedpack setaf +sharded shardfile shutil subclassing +symlink +symlinking +symlinks tensorspec tfdata tfdataset @@ -106,7 +122,9 @@ tfrecord tfrecords tinyaes tobytes +unbatched uoffset +uvx vmap vmaps xored diff --git a/skills.sh.json b/skills.sh.json new file mode 100644 index 00000000..acb15a39 --- /dev/null +++ b/skills.sh.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://skills.sh/schemas/skills.sh.schema.json", + "paths": [ + "./src/sedpack/.agents/skills" + ] +} diff --git a/src/sedpack/.agents/skills/sedpack/SKILL.md b/src/sedpack/.agents/skills/sedpack/SKILL.md new file mode 100644 index 00000000..e41bd2b8 --- /dev/null +++ b/src/sedpack/.agents/skills/sedpack/SKILL.md @@ -0,0 +1,495 @@ +--- +name: sedpack +description: >- + Creates, writes, inspects, and iterates multi-shard machine learning and + side-channel analysis datasets using the sedpack library (sedpack.io). Use + when creating a sedpack Dataset, defining Metadata, DatasetStructure, or + Attribute schemas, writing examples with dataset.filler(), DatasetFiller, + or dataset.write_multiprocessing(), iterating data for JAX, PyTorch, Keras, + TensorFlow, or NumPy (as_numpy_iterator, as_numpy_iterator_concurrent, + as_numpy_iterator_async, as_numpy_iterator_rust, + as_numpy_iterator_rust_batched, as_tfdataset), filtering or balancing + shards via custom_metadata, or converting raw datasets (HDF5, NPZ, traces) + into sedpack format. Don't use for modifying sedpack's internal Rust or + FlatBuffers compiler implementation unless high-level API semantics are + needed. +license: Apache-2.0 +--- + +# Using `sedpack` (`sedpack.io`) + +`sedpack` stores and streams sharded datasets using a hierarchical metadata tree +(`dataset_info.json` to `/shards_list.json` to shard files) backed by +FlatBuffers (`"fb"`), NPZ (`"npz"`), or TFRecords (`"tfrec"`). + +## 1. Core Imports + +Top-level `import sedpack` only exposes `sedpack.__version__`. Import API +classes from `sedpack.io` and type/split constants from `sedpack.io.types`: + +```python +import itertools +from pathlib import Path +import numpy as np + +from sedpack.io import ( + Attribute, Dataset, DatasetFiller, DatasetStructure, Metadata, +) +from sedpack.io.errors import DatasetExistsError +from sedpack.io.iteration import RustBatchedGenerator, RustGenerator +from sedpack.io.shard_file_metadata import ShardInfo, ShardListInfo +from sedpack.io.shard_info_iterator import CachedShardInfoIterator +from sedpack.io.shard_info_iterator.balanced_iterator import ( + BalancedShardInfoIterator, +) +from sedpack.io.types import ( + HOLDOUT_SPLIT, TEST_SPLIT, TRAIN_SPLIT, BatchT, ExampleT, SplitT, +) +``` + +## 2. Creating vs. Loading a Dataset + +- **Create a new dataset**: Call + `Dataset.create(path=dataset_path, metadata=metadata, ...)` + - Creates `dataset_path` and writes `dataset_info.json`. + - Raises `DatasetExistsError` (a subclass of `FileExistsError`) if + `dataset_info.json` already exists at `dataset_path`. +- **Load an existing dataset (to read or append)**: Call + `dataset = Dataset(dataset_path)` + - There is **no** `Dataset.load()` or `Dataset.open()` method. Always call + `Dataset(path)`. +- **Valid Splits (`SplitT`)**: + - Strictly `"train"`, `"test"`, or `"holdout"` (`TRAIN_SPLIT`, + `TEST_SPLIT`, `HOLDOUT_SPLIT`). + - **Gotcha**: Passing `"val"` or `"validation"` raises + `ValueError: Not a known split value`. In `sedpack`, use `"test"` for + validation during training and `"holdout"` for final evaluation. + +## 3. Defining the Schema (`Metadata`, `DatasetStructure`, `Attribute`) + +```python +metadata = Metadata( + description="AES power traces dataset", + dataset_license="Apache-2.0", + dataset_version="1.0.0", + custom_metadata={"capture_scope": "ChipWhisperer"}, +) + +dataset_structure = DatasetStructure( + saved_data_description=[ + Attribute(name="trace", dtype="float32", shape=(5000,)), + Attribute(name="plaintext", dtype="uint8", shape=(16,)), + Attribute(name="key_id", dtype="int", shape=()), + Attribute(name="label", dtype="uint8", shape=()), + Attribute(name="raw_packet", dtype="bytes", shape=()), + Attribute(name="fixed_nonce", dtype="bytes", shape=(12,)), + Attribute(name="notes", dtype="str", shape=()), + ], + compression="LZ4", # Default is "GZIP"; "LZ4" or "ZSTD" recommended for speed + examples_per_shard=256, # Default is 256 + shard_file_type="fb", # "fb" (default, fastest), "npz", or "tfrec" + hash_checksum_algorithms=("sha256",), +) + +dataset_path = Path("datasets/aes_traces") +dataset = Dataset.create( + path=dataset_path, + metadata=metadata, + dataset_structure=dataset_structure, +) +``` + +### `Attribute` `dtype` and `shape` Rules + +1. **Positive dimensions only**: Every integer in `shape` must be **strictly + positive (`> 0`)**. Passing `(-1,)`, `(0,)`, or `(None,)` raises + `ValueError`. +2. **Use `shape=()` for scalars and variable-length types**: + - NumPy scalar: `Attribute(name="label", dtype="uint8", shape=())` + - Arbitrary 64-bit signed Python `int`: + `Attribute(name="index", dtype="int", shape=())` + - UTF-8 variable-length string: + `Attribute(name="notes", dtype="str", shape=())` + - Variable-length byte sequence: + `Attribute(name="raw_packet", dtype="bytes", shape=())` + (`has_variable_size() == True`) +3. **Fixed-size vs. variable-size `"bytes"`**: + - `dtype="bytes", shape=()`: Variable-length `bytes`. + - `dtype="bytes", shape=(N,)`: Fixed-length `N`-byte sequence + (`has_variable_size() == False`). On write, accepts + `bytes`/`bytearray`/`memoryview` of length `N` or a 1D integer array of + shape `(N,)` with values in `[0, 255]`. On read, always returns `bytes` + of length `N`. +4. **Supported `dtype` strings**: + - Numeric/boolean arrays or scalars: `"float16"`, `"float32"`, + `"float64"`, `"int8"`, `"uint8"`, `"int16"`, `"uint16"`, `"int32"`, + `"uint32"`, `"int64"`, `"uint64"`, `"bool"`. + - Special scalar/sequence types: `"int"` (`shape=()`), `"str"` + (`shape=()`), `"bytes"` (`shape=()` or `shape=(N,)`). + +### Format (`shard_file_type`) & Compression Compatibility Matrix + +Selecting an unsupported `(shard_file_type, compression)` pair raises an +`AssertionError` or `ValueError`: + +| `shard_file_type` | Supported `compression` values | Key Notes | +| :--- | :--- | :--- | +| `"fb"` (FlatBuffers, default) | `""`, `"LZ4"`, `"ZSTD"`, `"GZIP"`, `"ZLIB"`, `"BZ2"`, `"LZMA"` | **Recommended**. Rust iterators (`as_numpy_iterator_rust*`) support `""`, `"LZ4"`, `"ZSTD"`, `"GZIP"`, `"ZLIB"` (do **not** use `"BZ2"` or `"LZMA"` with Rust iterators). Does **not** support `"ZIP"`. | +| `"npz"` (NumPy Zip) | `""`, `"ZIP"` | **Gotcha**: Default `compression="GZIP"` fails with `shard_file_type="npz"`. You **must** pass `compression="ZIP"` or `compression=""`. | +| `"tfrec"` (TFRecord) | `""`, `"GZIP"`, `"ZLIB"` | Requires `tensorflow` (`sedpack[tf]` extra). Note: `"str"` attributes are returned as UTF-8 `bytes` when read via `as_numpy_iterator*`. | + +## 4. Writing Examples to a Dataset + +### Strict `dtype` and `shape` Checking on Write + +For `"fb"` shards, `write_example` enforces +`np.can_cast(value, attribute.dtype, casting="safe")` and exact `shape` equality +(`np.asarray(value).shape == attribute.shape`). + +- **Gotcha**: Passing a default NumPy `float64` array to a `"float32"` + attribute, or a Python/NumPy `int64` to a `"uint8"`/`"int32"` attribute, + raises `ValueError: Cannot cast value of dtype ...`. +- **Fix**: Explicitly cast values before writing (for example, + `trace.astype(np.float32)`, `np.asarray(label, dtype=np.uint8)`). + +### Pattern A: Standard Single-Coordinator Writing (`dataset.filler`) + +Use `with dataset.filler(concurrency=4) as dataset_filler:` to stream examples. +Full shards are serialized, compressed, and hashed asynchronously in a +background `ProcessPoolExecutor` with `max(1, concurrency)` workers. + +```python +def make_example(i: int) -> ExampleT: + return { + "trace": np.zeros((5000,), dtype=np.float32), + "plaintext": np.zeros((16,), dtype=np.uint8), + "key_id": i, + "label": np.asarray(i % 256, dtype=np.uint8), + "raw_packet": b"variable_len_bytes", + "fixed_nonce": b"0123456789ab", + "notes": f"example_{i}", + } + +with dataset.filler(concurrency=4) as dataset_filler: + for i in range(500): + dataset_filler.write_example( + values=make_example(i), + split=TRAIN_SPLIT, + custom_metadata={"device": i % 2}, # Optional per-shard metadata + ) +``` + +- **Shard buffering & `custom_metadata` grouping**: + - Examples with different `custom_metadata` (or `None`/`{}` vs. non-empty + `custom_metadata`) are never mixed in the same shard. + - Up to `max(1, concurrency)` open shards are buffered in memory + simultaneously across `(split, custom_metadata)` pairs using an LRU + policy. Grouping writes by `(split, custom_metadata)` avoids flushing + partially filled shards when writing many distinct `custom_metadata` + values. +- **Appending to an existing dataset**: + - Load `dataset = Dataset(path)` and open another + `with dataset.filler() as filler:` block. Exiting the `with` block + merges the new shards into `/shards_list.json` and + `dataset_info.json`. +- **Updating `dataset.metadata` or `Attribute.custom_metadata`**: + - `dataset.filler()` saves `dataset_info.json` on `__exit__`. If you + modify `dataset.metadata.custom_metadata` or + `dataset.dataset_structure.saved_data_description[i].custom_metadata` + *after* the `with dataset.filler():` block (for example, recording + global min/max statistics), you **must** call `dataset.write_config()` + afterwards to persist those updates. +- **Converting existing NPZ, HDF5, or raw trace files**: + - Stream records into `with dataset.filler(concurrency=4) as filler:` and + cast each array to the schema `dtype` (`np.asarray(x, dtype=...)`). + - When reading `.npz`/`.npy` files, pass `allow_pickle=False` + (`with np.load(npz_path, allow_pickle=False) as archive:`) to prevent + unsafe `pickle` deserialization. For HDF5 (`h5py.File(h5_path, "r")`), + slice rows incrementally rather than loading the entire file into RAM. + +### Pattern B: Built-In Multiprocessing (`dataset.write_multiprocessing`) + +When generating or converting examples is CPU-bound, use +`dataset.write_multiprocessing`. Each worker automatically receives an isolated +`DatasetFiller` with a unique `relative_path_from_split`. + +```python +def feed_worker( + dataset_filler: DatasetFiller, + worker_items: list[int], + *, + split: SplitT, +) -> int: + # MUST enter the context manager inside feed_writer + with dataset_filler: + for item in worker_items: + dataset_filler.write_example(values=make_example(item), split=split) + return len(worker_items) + +# Spawn 2 workers (len(custom_arguments) determines the number of tasks) +counts = dataset.write_multiprocessing( + feed_writer=feed_worker, + custom_arguments=[([0, 1, 2],), ([3, 4, 5],)], + custom_kwarguments=[{"split": TRAIN_SPLIT}, {"split": TEST_SPLIT}], + consistency_check=True, + single_process=False, +) +``` + +- **Gotcha**: The first positional argument of `feed_writer` is **always** + `dataset_filler: DatasetFiller`, and `feed_writer` **must** use + `with dataset_filler:`. + +### Pattern C: Custom Multi-Worker / Distributed Writing + +If you manage your own multi-process or distributed workers: + +1. Every worker must open `Dataset(dataset_path)` and construct `DatasetFiller` + with a **non-empty, mutually disjoint** `relative_path_from_split` (no + worker path may be a parent prefix of another; for example, use + `Path("worker_0")`, `Path("worker_1")`). +2. Each worker writes inside `with filler:` and returns + `filler.get_updated_infos()`. +3. After all workers finish, a single coordinator calls + `dataset.write_config(updated_infos=all_updated_infos)`. + +```python +# Inside worker `worker_id`: +worker_ds = Dataset(dataset_path) +filler = DatasetFiller( + dataset=worker_ds, + concurrency=2, + relative_path_from_split=Path(f"worker_{worker_id}"), +) +with filler: + filler.write_example(values=make_example(worker_id), split=TRAIN_SPLIT) +worker_infos: list[ShardListInfo] = filler.get_updated_infos() + +# On the coordinator after all workers complete: +dataset.write_config(updated_infos=all_worker_infos) +``` + +> **Critical**: Never use default `relative_path_from_split=Path(".")` across +> concurrent workers, even if workers write to different splits—exiting a root +> `DatasetFiller` immediately writes `dataset_info.json` and overwrites other +> splits. + +## 5. Reading & Iterating a Dataset + +### Critical Defaults Gotcha: `repeat=True` and `shuffle=1_000` + +Every `dataset.as_*` iterator defaults to **`repeat=True` (infinite stream)** +and **`shuffle=1_000`**: + +- **For training**: Keep `repeat=True, shuffle=1_000` (or larger) and bound + epochs with `itertools.islice(it, steps_per_epoch)` or `steps_per_epoch` in + `model.fit`. +- **For evaluation, validation, testing, or single-pass statistics (such as + SNR or accuracy)**: You **must** pass **`repeat=False, shuffle=0`**. + Otherwise a `for batch in dataset.as_numpy_iterator...(split=...)` loop + runs forever. +- **Keyword-only `split`**: Always pass `split=` as a keyword argument + (`split=TRAIN_SPLIT`). + +### Choosing the Right Iterator + +| Method | Yields / Returns | Formats | Record Transform Arg | When to Use | +| :--- | :--- | :--- | :--- | :--- | +| `as_numpy_iterator_rust_batched` | `Iterator[BatchT]` | `"fb"` (`""`, `"LZ4"`, `"ZSTD"`, `"GZIP"`, `"ZLIB"`) | `process_batch` | **Best for JAX, PyTorch, Keras, and fast batched evaluation**. | +| `as_numpy_iterator_rust` | `Iterator[ExampleT]` | `"fb"` (`""`, `"LZ4"`, `"ZSTD"`, `"GZIP"`, `"ZLIB"`) | `process_record` | Fast unbatched iteration over `"fb"` shards. | +| `as_tfdataset` | `tf.data.Dataset` | `"tfrec"`, `"fb"`, `"npz"` | `process_record` | TensorFlow / `tf.data` pipelines. | +| `as_numpy_iterator` | `Iterator[ExampleT]` | `"fb"`, `"npz"`, `"tfrec"` | `process_record` | Simple single-process iteration, debugging, or `"npz"`/`"tfrec"` reading. | +| `as_numpy_iterator_concurrent` | `Iterator[ExampleT]` | `"fb"`, `"npz"`, `"tfrec"` | `process_record` | Multi-worker Python unbatched iteration (supports `custom_metadata_type_limit`). | +| `as_numpy_iterator_async` | `AsyncIterator[ExampleT]` | `"fb"`, `"npz"` | `process_record` | `asyncio` pipelines (`async for example in ...`). | + +### 1. Fast Batched Iteration (`as_numpy_iterator_rust_batched`) + +```python +# Training (infinite shuffled stream of batches) +train_batches = dataset.as_numpy_iterator_rust_batched( + split=TRAIN_SPLIT, + batch_size=64, + repeat=True, + shuffle=10_000, + process_batch=lambda b: (b["trace"], b["label"]), +) + +# Single-pass evaluation (finite, deterministic order) +for batch in dataset.as_numpy_iterator_rust_batched( + split=HOLDOUT_SPLIT, + batch_size=64, + repeat=False, + shuffle=0, +): + traces = batch["trace"] # np.ndarray of shape (64, 5000), dtype float32 + labels = batch["label"] # np.ndarray of shape (64,), dtype uint8 + packets = batch["raw_packet"] # list[bytes] of length 64 + notes = batch["notes"] # list[str] of length 64 + key_ids = batch["key_id"] # list[int] of length 64 +``` + +- **Always pass `batch_size` explicitly**: `as_numpy_iterator_rust_batched` + defaults to `batch_size=1` (unlike `as_tfdataset`, which defaults to + `batch_size=32`). +- **Batched value types (`BatchT`)**: + - Numeric/boolean NumPy attributes have shape + `(batch_size, *attribute.shape)`. + - `"bytes"`, `"str"`, and `"int"` attributes are returned as + `list[bytes]`, `list[str]`, and `list[int]` of length `batch_size`. +- **Trailing remainder batches**: When `repeat=False`, if the total number of + examples `N` is not divisible by `batch_size`, the trailing + `N % batch_size` examples are dropped by `as_numpy_iterator_rust_batched`. + To visit every single example when `N % batch_size != 0`, use unbatched + `as_numpy_iterator_rust(split=..., repeat=False, shuffle=0)`. + +### 2. Unbatched Iteration (`as_numpy_iterator_rust`, `as_numpy_iterator`, `as_numpy_iterator_concurrent`) + +```python +for example in dataset.as_numpy_iterator_rust( + split=TEST_SPLIT, + repeat=False, + shuffle=0, + process_record=None, +): + trace = example["trace"] # np.ndarray of shape (5000,) + label = example["label"] # np.ndarray scalar of shape () +``` + +- **Pickleable `process_record` in `as_numpy_iterator_concurrent`**: When + `shuffle > 0`, `as_numpy_iterator_concurrent` executes `process_record` + inside `multiprocessing.Pool` workers, so `process_record` **must be a + top-level function or `functools.partial`** (not a `lambda` or local + closure). + +### 3. TensorFlow Dataset (`as_tfdataset`) + +```python +tf_ds = dataset.as_tfdataset( + split=TRAIN_SPLIT, + batch_size=32, # Pass batch_size=0 to disable batching + repeat=True, + shuffle=1_000, +) +# When reshaping or converting dictionaries to (x, y) tuples for Keras on "fb" +# or "npz" datasets, apply .map() on the returned tf.data.Dataset rather than +# changing structure inside process_record (which must match +# saved_data_description): +tf_ds = tf_ds.map(lambda ex: (ex["trace"], ex["label"])) +``` + +## 6. Inspecting Metadata, Filtering, Balancing, & Checking Integrity + +### Inspect Example & Shard Counts Without Reading Shards + +```python +dataset = Dataset(dataset_path) + +# Check if split exists before indexing dataset.dataset_info.splits +if TRAIN_SPLIT in dataset.dataset_info.splits: + split_meta = dataset.dataset_info.splits[TRAIN_SPLIT] + print(split_meta.number_of_examples, split_meta.number_of_shards) + +# Look up an Attribute schema by name +trace_attr = dataset.dataset_structure.attribute_by_name("trace") +print(trace_attr.dtype, trace_attr.shape, trace_attr.custom_metadata) +``` + +### Filter or Limit Shards by `ShardInfo` Metadata + +All `as_*` iterators accept `shard_filter: Callable[[ShardInfo], bool]` and +`shards: int | None` (evaluated purely from `shards_list.json` without reading +shard files): + +```python +device_0_it = dataset.as_numpy_iterator_rust_batched( + split=TRAIN_SPLIT, + batch_size=32, + repeat=False, + shuffle=0, + shards=10, # Optional: read at most the first 10 matching shards + shard_filter=lambda shard_info: shard_info.custom_metadata.get("device") == 0, +) +``` + +- **Empty filter result**: If `shard_filter` matches zero shards (or the + split has zero shards), `as_numpy_iterator`, `as_numpy_iterator_rust`, and + `as_numpy_iterator_rust_batched` yield zero items, whereas + `as_numpy_iterator_concurrent`, `as_numpy_iterator_async`, `as_tfdataset`, + and `shard_paths_dataset` raise + `ValueError: The list of shards is empty. Try less restrictive filtering.` +- **Limiting shards per distinct `custom_metadata`**: `as_numpy_iterator`, + `as_numpy_iterator_concurrent`, and `as_tfdataset` also accept + `custom_metadata_type_limit=K` to read at most `K` shards for each unique + `ShardInfo.custom_metadata` dictionary. +- **Iterating shard metadata directly**: Use + `for shard_info in dataset.shard_info_iterator(TRAIN_SPLIT): ...` (pass + `None` to iterate across all present splits). + +### Balanced Iteration Across Shard Categories (`BalancedShardInfoIterator`) + +To sample shards with equal (or custom-weighted) probability across metadata +attributes regardless of shard count imbalance: + +```python +def by_device(shard_info: ShardInfo) -> int: + return shard_info.custom_metadata["device"] + +# Optional custom weight per category: +# by_device.weight = lambda shard_info: 3.0 if shard_info.custom_metadata["device"] == 0 else 1.0 + +balanced_shards = BalancedShardInfoIterator( + dataset_path=dataset.path, + dataset_info=dataset.dataset_info, + split=TRAIN_SPLIT, + repeat=True, + shuffle=100, + balance_by=(by_device,), + shard_filter=None, +) + +# Option A: Fast Rust batched iteration over balanced shards ("fb" format) +with RustBatchedGenerator( + dataset_path=dataset.path, + dataset_structure=dataset.dataset_structure, + shard_iterator=balanced_shards, + batch_size=64, + process_batch=None, + shuffle_buffer_size=1_000, +) as rust_generator: + for batch in itertools.islice(rust_generator(), 100): + traces = batch["trace"] + +# Option B: Python unbatched iteration over balanced shards (all formats) +balanced_examples = dataset.example_iterator_from_shard_iterator( + shard_iterator=balanced_shards, + process_record=None, + shuffle=1_000, +) +for example in itertools.islice(balanced_examples, 100): + trace = example["trace"] +``` + +### Verify Dataset Integrity (`dataset.check()`) + +```python +# Validates dataset_info.json, all shards_list.json files, shard hashes, and example counts +dataset.check(show_progressbar=False) + +# Get root dataset_info.json checksums (call after write_config) +checksums: tuple[str, ...] = dataset.current_metadata_checksums() +``` + +## 7. Quick Troubleshooting Table + +| Error / Symptom | Cause | Solution | +| :--- | :--- | :--- | +| Evaluation loop hangs / never finishes | Default `repeat=True` creates an infinite iterator | Pass `repeat=False, shuffle=0` to `dataset.as_*` | +| `ValueError: Not a known split value` | Passed `"val"` or `"validation"` as `split` | Use `"train"`, `"test"` (`TEST_SPLIT`), or `"holdout"` (`HOLDOUT_SPLIT`) | +| `ValueError: The list of shards is empty. Try less restrictive filtering.` | `shard_filter` matched zero shards or split has no shards | Relax `shard_filter` or check `split in dataset.dataset_info.splits` first | +| `ValueError: Cannot cast value of dtype ...` | Passed `float64` to `"float32"` or `int64` to `"uint8"`/`"int32"` in `"fb"` writer | Cast explicitly before `write_example`: `trace.astype(np.float32)` | +| `ValueError: Shape requires positive values` | Used `shape=()` with `0` or `-1`, such as `(-1, 128)` or `(0,)` | Use `shape=()` for scalars/variable-length `"str"`/`"bytes"`, or fixed positive dimensions | +| `AssertionError` when creating/writing `"npz"` dataset | Default `compression="GZIP"` is unsupported for `shard_file_type="npz"` | Set `compression="ZIP"` or `compression=""` in `DatasetStructure` | +| `TypeError: ... takes 1 positional argument` on `as_numpy_iterator*` | Passed `split` by position (`dataset.as_numpy_iterator("train")`) | Pass `split` as a keyword argument (`dataset.as_numpy_iterator(split="train")`) | +| `TypeError: ... got an unexpected keyword argument 'process_record'` on `as_numpy_iterator_rust_batched` | `as_numpy_iterator_rust_batched` transforms batches, not individual records | Use `process_batch=...` instead of `process_record=...` | +| Attribute `custom_metadata` changes lost after `with dataset.filler():` | `dataset.filler()` writes `dataset_info.json` upon exiting the `with` block | Call `dataset.write_config()` after mutating `dataset.metadata` or `Attribute.custom_metadata` | +| `AttributeError: Can't pickle local object` in `as_numpy_iterator_concurrent` | Passed a `lambda` or closure as `process_record` while `shuffle > 0` | Define `process_record` as a top-level function or use `as_numpy_iterator_rust` | diff --git a/tests/io/test_skills.py b/tests/io/test_skills.py new file mode 100644 index 00000000..241954e3 --- /dev/null +++ b/tests/io/test_skills.py @@ -0,0 +1,107 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for the bundled sedpack Agent Skill and skills.sh.json.""" + +import ast +from importlib import resources +import json +from pathlib import Path +import re + + +def test_bundled_skill_location_and_content() -> None: + """Verify SKILL.md is bundled at sedpack/.agents/skills/sedpack/SKILL.md.""" + skill_dir = resources.files("sedpack") / ".agents" / "skills" / "sedpack" + skill_md = skill_dir / "SKILL.md" + assert skill_dir.is_dir() + assert skill_md.is_file() + + text = skill_md.read_text(encoding="utf-8") + raw_bytes = skill_md.read_bytes() + line_count = len(text.splitlines()) + assert 50 <= line_count <= 500 + assert len(raw_bytes) <= 35_000 + + assert text.startswith("---\n") + end_idx = text.find("\n---\n", 4) + assert end_idx != -1, "Missing closing YAML frontmatter delimiter" + frontmatter = text[4:end_idx] + + name_match = re.search(r"^name:\s*(\S+)\s*$", frontmatter, re.MULTILINE) + assert name_match is not None + skill_name = name_match.group(1) + assert skill_name == "sedpack" + assert re.fullmatch(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$", skill_name) + + assert "description: >-\n" in frontmatter + desc_lines: list[str] = [] + in_desc = False + for line in frontmatter.splitlines(): + if line.startswith("description:"): + in_desc = True + continue + if in_desc: + if line.startswith(" "): + desc_lines.append(line.strip()) + else: + break + description = " ".join(desc_lines) + assert 0 < len(description) <= 1024 + assert "<" not in description and ">" not in description + assert "Use when" in description + assert "Don't use for" in description + assert "/tmp/" not in text + + for expected_symbol in ( + "Dataset", + "Metadata", + "DatasetStructure", + "Attribute", + "as_numpy_iterator", + "as_tfdataset", + "write_multiprocessing", + "BalancedShardInfoIterator", + "allow_pickle=False", + ): + assert expected_symbol in text + + python_blocks = re.findall( + r"```python\n(.*?)```", + text, + flags=re.DOTALL, + ) + assert len(python_blocks) >= 10 + for block in python_blocks: + ast.parse(block) + + +def test_skills_sh_json_config() -> None: + """Verify skills.sh.json follows the skills.sh schema and resolves.""" + repo_root = Path(__file__).resolve().parents[2] + config_path = repo_root / "skills.sh.json" + assert config_path.is_file() + + config = json.loads(config_path.read_text(encoding="utf-8")) + assert (config.get("$schema") == + "https://skills.sh/schemas/skills.sh.schema.json") + paths = config.get("paths") + assert isinstance(paths, list) and paths == ["./src/sedpack/.agents/skills"] + + for rel_path in paths: + assert isinstance(rel_path, str) + assert rel_path.startswith("./") + assert "\\" not in rel_path + container = repo_root / rel_path + assert container.is_dir() + assert (container / "sedpack" / "SKILL.md").is_file() diff --git a/website/src/content/docs/start_here/install.mdx b/website/src/content/docs/start_here/install.mdx index 12291fa1..ee8ee005 100644 --- a/website/src/content/docs/start_here/install.mdx +++ b/website/src/content/docs/start_here/install.mdx @@ -39,7 +39,7 @@ Development dependencies: Clone and create a Python virtual environment: ```bash -git clone github.com/google/sedpack/ # Clone the repository +git clone https://github.com/google/sedpack # Clone the repository python3 -m venv my_env # Create Python virtual environment source my_env/bin/activate # Activate your virtual environment cd sedpack/ # Change directory to the cloned git repository @@ -106,3 +106,179 @@ python -m pytest ```bash maturin develop --release ``` + +## AI Agents and LLM Skills + +Sedpack ships with instructions for both AI coding assistants using `sedpack` +in downstream projects (`SKILL.md`) and agents contributing to the `sedpack` +repository itself (`AGENTS.md`). + +### Using `sedpack` with Coding Agents (`SKILL.md`) + +The high-level `sedpack.io` API skill is bundled inside the `sedpack` Python +package at `src/sedpack/.agents/skills/sedpack/SKILL.md` on +[GitHub](https://github.com/google/sedpack/blob/main/src/sedpack/.agents/skills/sedpack/SKILL.md). +Installing `sedpack` never mutates your home directory (`~/`) implicitly; use +standard skill managers below to link or copy the skill. + +#### Project-Local vs. Global (`~/`) Installation + +| Approach | Auto-Updates on Upgrade? | Multi-Project `.venv` Safety | Resilience to `.venv` Rebuilds | Recommended When | +| :--- | :--- | :--- | :--- | :--- | +| **Project-Local Symlink** (`.agents/skills/sedpack` -> `.venv/...`) | Yes | High (matches the project's `.venv` version) | High (relative link resolves once `.venv` is installed) | **Default** for Python projects using virtual environments | +| **Home-Directory Symlink** (`~/.agents/skills/sedpack` -> path) | Yes | Low (single global version across all repos) | Low (breaks if target `.venv` or Python minor version changes) | Editable local checkouts (`pip install -e`) or global environments | +| **File Copy** (`--copy` to `.agents/skills/sedpack`) | No (re-run after upgrade) | High | High (no external target dependency) | Windows without Developer Mode symlink permissions | + +- **Option 1 — Link from your Python environment via + [Library Skills (`library-skills`)](https://library-skills.io/) (recommended + default)**: After installing `sedpack` into your project's environment + (`venv`, `virtualenv`, `uv`, `pdm`, `poetry`, `pipenv`, `hatch`, or + `conda`), `library-skills` scans `site-packages` (and editable installs) + and creates a relative directory symlink in `./.agents/skills/sedpack`: + + ```bash + # Interactive discovery and installation (or use `pipx run library-skills`): + uvx library-skills + + # Non-interactive install of the sedpack skill into .agents/skills/sedpack: + uvx library-skills install --skill sedpack --yes + + # Also link into .claude/skills/sedpack, or copy files instead of symlinking: + uvx library-skills install --skill sedpack --yes --claude + uvx library-skills install --skill sedpack --yes --copy + + # List discovered/installed skills, verify symlinks in CI, or repair links: + uvx library-skills list + uvx library-skills --check + uvx library-skills --yes + + # Remove the managed symlink: + uvx library-skills remove sedpack + ``` + + `library-skills` automatically discovers `.venv`, `UV_PROJECT_ENVIRONMENT`, + and `CONDA_PREFIX` environments and creates a relative directory symlink to + `sedpack/.agents/skills/sedpack` so running `pip install -U sedpack` updates + the skill automatically. If your environment manager stores virtual + environments outside the project root (such as Poetry's default cache, + Pipenv without `PIPENV_VENV_IN_PROJECT=1`, or Hatch), pass its path via + `UV_PROJECT_ENVIRONMENT`: + + ```bash + UV_PROJECT_ENVIRONMENT="$(poetry env info --path)" uvx library-skills install --skill sedpack --yes + UV_PROJECT_ENVIRONMENT="$(pipenv --venv)" uvx library-skills install --skill sedpack --yes + UV_PROJECT_ENVIRONMENT="$(hatch env find)" uvx library-skills install --skill sedpack --yes + ``` + +- **Option 2 — Install from GitHub via + [Agent Skills CLI (`npx skills`)](https://skills.sh/)**: Reads + `skills.sh.json` at the repository root and installs the `sedpack` skill + directly from `google/sedpack` (or a local checkout) into project-local + (`.agents/skills/`) or global (`~/.agents/skills/` with `-g`) directories + across 40+ coding agents: + + ```bash + # Interactive project-local install from GitHub: + npx skills add google/sedpack + + # Non-interactive project-local install (or target a specific agent): + npx skills add google/sedpack --skill sedpack -y + npx skills add google/sedpack --skill sedpack --agent gemini-cli -y + npx skills add google/sedpack --skill sedpack --agent claude-code -y + + # Global user-level install into ~/.agents/skills/sedpack: + npx skills add google/sedpack --skill sedpack -g -y + + # List available skills in the repository, or check/update installed skills: + npx skills add google/sedpack --list + npx skills check + npx skills update + ``` + +- **Option 3 — Link with Gemini CLI**: Link the bundled skill directory + directly from the active Python environment (into `~/.gemini/skills` by + default, or `.gemini/skills` with `--scope workspace`): + + ```bash + gemini skills link "$(python -c 'import importlib.resources as r; print(r.files("sedpack") / ".agents/skills/sedpack")')" + ``` + +#### Updating Skills When `sedpack` Updates + +How your agent skill stays in sync when you upgrade `sedpack` depends on how it +was installed: + +| Installation Method | Scope | Updates on `pip install -U sedpack`? | What Happens on `.venv` Rebuild / Python Upgrade? | How to Update or Repair | +| :--- | :--- | :--- | :--- | :--- | +| **`uvx library-skills` (symlink)** | Project-local (`.agents/skills/sedpack`) | **Automatic** (points directly into `.venv` `site-packages`) | Resolves automatically if same Python `X.Y`; breaks if `python3.12` -> `python3.13` | `uvx library-skills --yes` (only needed after Python minor version change) | +| **`uvx library-skills --copy`** | Project-local (`.agents/skills/sedpack`) | **Manual** (static file copy) | Unaffected (files live in repo) | `rm -rf .agents/skills/sedpack && uvx library-skills install --skill sedpack --yes --copy` | +| **Editable checkout (`pip install -e .`)** | Project-local or Global (`~/`) | **Automatic** on `git pull` / branch switch | Unaffected (points to git source tree) | `git pull` | +| **`npx skills add google/sedpack`** | Project-local or Global (`-g` in `~/.agents/skills`) | **Manual** (tracks GitHub `main`, decoupled from `.venv`) | Unaffected (copied from GitHub) | `npx skills check` and `npx skills update` | +| **`gemini skills link`** | Global (`~/.gemini/skills`) or Local (`--scope workspace`) | **Automatic** for in-place upgrade in the linked `.venv` | Breaks if the linked `.venv` is deleted or changes Python `X.Y` | Run `/skills reload` in Gemini CLI, or re-run `gemini skills link` if `.venv` moved | + +- **Automatic updates with project-local `.venv` symlinks (`library-skills`)**: + When linked via `uvx library-skills`, `.agents/skills/sedpack` is a + directory symlink to `site-packages/sedpack/.agents/skills/sedpack`. Running + `pip install -U sedpack`, `uv sync --upgrade-package sedpack`, + `poetry update sedpack`, or `conda update` updates `SKILL.md` (and any new + companion files) immediately with zero extra steps, while keeping each + project matched to its own `.venv` version. +- **Repairing `.venv` symlinks after a Python minor version upgrade**: + On Linux and macOS, `site-packages` lives under `.venv/lib/pythonX.Y/`. If + you recreate `.venv` with the *same* Python `X.Y` version, the relative + symlink resolves again as soon as `sedpack` is installed. If you upgrade + from `python3.12` to `python3.13`, repair the symlink (and check symlink + health in CI) with: + + ```bash + uvx library-skills --yes + uvx library-skills --check + ``` + +- **Manually updating a copied skill (`library-skills --copy`)**: + Because `--copy` writes regular files instead of a symlink, re-copy the + skill after upgrading `sedpack`: + + ```bash + rm -rf .agents/skills/sedpack + uvx library-skills install --skill sedpack --yes --copy + ``` + +- **Manually updating GitHub-installed skills (`npx skills` local or global + `-g`)**: + `npx skills` installs from the `google/sedpack` GitHub repository and + records the commit hash in `skills-lock.json` rather than reading `.venv`. + Note that a global (`-g`) installation places a single version in + `~/.agents/skills/sedpack` shared across all repositories on your machine, + which can drift if different projects pin different `sedpack` versions. To + check for and pull the latest skill from GitHub: + + ```bash + npx skills check + npx skills update + ``` + +- **Reloading or re-linking in Gemini CLI (`gemini skills link`)**: + After an in-place `pip install -U sedpack` in the linked environment, run + `/skills reload` inside an active Gemini CLI session to pick up changes. If + you recreated the environment under a new path or Python minor version, + re-run `gemini skills link`. + +#### Cross-Platform and Directory Casing Notes + +- **Directory casing**: Always use lowercase `.agents/skills`, + `.gemini/skills`, and `.claude/skills` (case-sensitive Linux filesystems + will not discover `.Gemini/skills`). +- **Agent path conventions**: Project-local `.agents/skills/` is supported + across modern coding agents (and `.gemini/skills/` or `.claude/skills/` for + agent-specific local directories); for user-level global directories, + Gemini CLI reads `~/.gemini/skills/` and Claude Code reads + `~/.claude/skills/`. + +### Developing `sedpack` with Coding Agents (`AGENTS.md`) + +When working on the `sedpack` repository itself, coding agents automatically +read [`AGENTS.md`](https://github.com/google/sedpack/blob/main/AGENTS.md) at the +repository root for build, test (`pytest`), formatting (`yapf`, `cargo fmt`), +type-checking (`mypy`), linting (`pylint`, `clippy`, `markdownlint`), and +spell-checking (`cspell`) instructions. diff --git a/website/src/content/docs/start_here/intro.md b/website/src/content/docs/start_here/intro.md index de07338d..e411e114 100644 --- a/website/src/content/docs/start_here/intro.md +++ b/website/src/content/docs/start_here/intro.md @@ -9,3 +9,10 @@ See [code samples](https://github.com/google/sedpack/tree/main/docs). The code is a major refactor of the data saving and loading code from the [SCAAML](https://github.com/google/scaaml) project. + +Using an AI coding assistant or LLM agent? Sedpack ships with a bundled +`SKILL.md` for the high-level `sedpack.io` API and a repository +[`AGENTS.md`](https://github.com/google/sedpack/blob/main/AGENTS.md) for +contributors—see the +[Installation guide](/sedpack/start_here/install/#ai-agents-and-llm-skills) for +setup instructions.