Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
316 changes: 316 additions & 0 deletions .github/workflows/skills.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,316 @@
name: skills
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
types: [opened, synchronize, reopened]
paths:
- 'skills.sh.json'
- 'pyproject.toml'
- 'src/sedpack/.agents/skills/**'
- 'tests/io/test_skills.py'
- '.github/workflows/skills.yml'
merge_group: # Needed for required workflows
# Run after a review has been submitted (this is a required workflow which
# might not be triggered when no code changes -- trigger before going to
# merge queue).
pull_request_review:
types: [submitted]

jobs:
validate_skills:
name: Validate skills
runs-on: ubuntu-22.04
steps:
- name: Checkout the code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- name: Set up Python 3.11
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: '3.11'
cache: 'pip'
- name: Install schema and skill validators
run: |
python -m pip install --upgrade pip
pip install --upgrade check-jsonschema skills-ref pytest
- name: Validate skills.sh.json against schema
run: |
check-jsonschema --schemafile https://skills.sh/schemas/skills.sh.schema.json skills.sh.json
- name: Validate configured skill paths and SKILL.md files
run: |
python - <<'EOF'
import json
from pathlib import Path
from skills_ref import validate

config = json.loads(Path("skills.sh.json").read_text(encoding="utf-8"))
expected_schema = "https://skills.sh/schemas/skills.sh.schema.json"
assert config.get("$schema") == expected_schema, (
f"Expected $schema={expected_schema!r}, got {config.get('$schema')!r}"
)
paths = config.get("paths", [])
assert paths, "skills.sh.json must define non-empty 'paths'"

for rel_path in paths:
container = Path(rel_path)
assert container.is_dir(), (
f"Configured skills path does not exist: {rel_path}"
)
skill_dirs = [p.parent for p in sorted(container.rglob("SKILL.md"))]
assert skill_dirs, f"No SKILL.md found under {rel_path}"
for skill_dir in skill_dirs:
errors = validate(skill_dir)
assert not errors, f"Validation failed for {skill_dir}: {errors}"
print(f"Validated skill: {skill_dir}")
EOF
- name: Run unit tests for bundled skill metadata
run: |
PYTHONPATH=./src pytest tests/io/test_skills.py
- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: "24"
- name: Verify npx skills discovery and installation via skills.sh.json
run: |
npx --yes skills add . --list
npx --yes skills add . --skill sedpack --agent gemini-cli -y
test -f .agents/skills/sedpack/SKILL.md

env_managers:
name: Test environment managers
runs-on: ubuntu-22.04
steps:
- name: Checkout the code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- name: Set up Python 3.11
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97
with:
python-version: '3.11'
cache: 'pip'
- name: Setup Rust toolchain
uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f
- name: Build sedpack wheel and install environment managers + library-skills
run: |
python -m pip install --upgrade pip
pip install "maturin>=1.7,<2.0" library-skills virtualenv uv poetry pdm pipenv hatch
pip wheel --no-deps --no-build-isolation . -w "$RUNNER_TEMP/dist"
python - "$RUNNER_TEMP/dist" <<'EOF'
import sys, zipfile
from pathlib import Path
wheels = list(Path(sys.argv[1]).glob("sedpack-*.whl"))
assert len(wheels) == 1, wheels
with zipfile.ZipFile(wheels[0]) as zf:
names = zf.namelist()
assert "sedpack/.agents/skills/sedpack/SKILL.md" in names, names
rec = [n for n in names if n.endswith("RECORD")][0]
rec_text = zf.read(rec).decode("utf-8")
assert "sedpack/.agents/skills/sedpack/SKILL.md" in rec_text
EOF
- name: Test standard venv + library-skills (wheel and editable installs)
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/venv_test"
mkdir -p "$WORK/proj" "$WORK/edit_proj"

cd "$WORK/proj"
python -m venv .venv
.venv/bin/pip install --no-deps "$WHEEL"
library-skills install --skill sedpack --yes --claude
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
test -L .claude/skills/sedpack
test -f .claude/skills/sedpack/SKILL.md
python -c 'import os; link = os.readlink(".agents/skills/sedpack"); assert not os.path.isabs(link), link'
library-skills --check --claude

# Also test --copy mode
rm -rf .agents .claude
library-skills install --skill sedpack --yes --copy
test ! -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md

# Also test editable install discovery via direct_url.json
cd "$WORK/edit_proj"
python -m venv .venv
.venv/bin/pip install "maturin>=1.7,<2.0"
.venv/bin/pip install --no-deps --no-build-isolation -e "$GITHUB_WORKSPACE"
library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
library-skills --check
- name: Test virtualenv + library-skills
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/virtualenv_test"
mkdir -p "$WORK/proj"
cd "$WORK/proj"
virtualenv .venv
.venv/bin/pip install --no-deps "$WHEEL"
library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
python -c 'import os; link = os.readlink(".agents/skills/sedpack"); assert not os.path.isabs(link), link'
library-skills --check
- name: Test uv + library-skills
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/uv_test"
mkdir -p "$WORK/proj"
cd "$WORK/proj"
uv venv .venv
uv pip install --python .venv/bin/python --no-deps "$WHEEL"
library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
python -c 'import os; link = os.readlink(".agents/skills/sedpack"); assert not os.path.isabs(link), link'
library-skills --check
- name: Test poetry (in-project and external cache) + library-skills
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/poetry_test"
mkdir -p "$WORK/inproj" "$WORK/cache_proj" "$WORK/home"
export HOME="$WORK/home"

# 1. Poetry with virtualenvs.in-project = true
cd "$WORK/inproj"
cat > pyproject.toml <<'EOF'
[project]
name = "demo-poetry-inproj"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = ["sedpack"]
EOF
poetry config virtualenvs.in-project true --local
poetry env use python
poetry run pip install --no-deps "$WHEEL"
library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
library-skills --check

# 2. Poetry with default external cache virtualenv
cd "$WORK/cache_proj"
cat > pyproject.toml <<'EOF'
[project]
name = "demo-poetry-cache"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = ["sedpack"]
EOF
poetry config virtualenvs.in-project false --local
poetry env use python
poetry run pip install --no-deps "$WHEEL"
UV_PROJECT_ENVIRONMENT="$(poetry env info --path)" library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
UV_PROJECT_ENVIRONMENT="$(poetry env info --path)" library-skills --check
- name: Test pdm + library-skills
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/pdm_test"
mkdir -p "$WORK/proj" "$WORK/home"
export HOME="$WORK/home"
cd "$WORK/proj"
cat > pyproject.toml <<'EOF'
[project]
name = "demo-pdm"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = ["sedpack"]
EOF
pdm venv create --force python
PDM_PY=$(pdm info --python)
uv pip install --python "$PDM_PY" --no-deps "$WHEEL"
library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
library-skills --check
- name: Test pipenv (in-project and external virtualenv) + library-skills
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/pipenv_test"
mkdir -p "$WORK/inproj" "$WORK/ext_proj" "$WORK/home"
export HOME="$WORK/home"

# 1. Pipenv with PIPENV_VENV_IN_PROJECT=1
cd "$WORK/inproj"
export PIPENV_VENV_IN_PROJECT=1
pipenv --python "$(which python)"
PIPENV_PY=$(pipenv --py)
uv pip install --python "$PIPENV_PY" --no-deps "$WHEEL"
library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
library-skills --check
unset PIPENV_VENV_IN_PROJECT

# 2. Pipenv default external virtualenv
cd "$WORK/ext_proj"
export WORKON_HOME="$WORK/home/.virtualenvs"
pipenv --python "$(which python)"
PIPENV_PY=$(pipenv --py)
uv pip install --python "$PIPENV_PY" --no-deps "$WHEEL"
UV_PROJECT_ENVIRONMENT="$(pipenv --venv)" library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
UV_PROJECT_ENVIRONMENT="$(pipenv --venv)" library-skills --check
- name: Test hatch + library-skills
run: |
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/hatch_test"
mkdir -p "$WORK/proj" "$WORK/home"
export HOME="$WORK/home"
cd "$WORK/proj"
cat > pyproject.toml <<'EOF'
[project]
name = "demo-hatch"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = ["sedpack"]
EOF
hatch env create
HATCH_ENV_DIR=$(hatch env find)
uv pip install --python "$HATCH_ENV_DIR/bin/python" --no-deps "$WHEEL"
UV_PROJECT_ENVIRONMENT="$HATCH_ENV_DIR" library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
UV_PROJECT_ENVIRONMENT="$HATCH_ENV_DIR" library-skills --check
- name: Test conda (prefix and named environments) + library-skills
run: |
if ! command -v conda >/dev/null 2>&1; then
if [ -x "$CONDA/bin/conda" ]; then
export PATH="$CONDA/bin:$PATH"
else
echo "conda not installed on runner; skipping conda step"
exit 0
fi
fi
WHEEL=$(ls "$RUNNER_TEMP/dist"/sedpack-*.whl)
WORK="$RUNNER_TEMP/conda_test"
mkdir -p "$WORK/prefix_proj" "$WORK/named_proj"

# 1. Local prefix environment (./.conda)
cd "$WORK/prefix_proj"
conda create --yes --quiet --prefix ./.conda python=3.11 pip
./.conda/bin/pip install --no-deps "$WHEEL"
CONDA_PREFIX="$WORK/prefix_proj/.conda" library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
CONDA_PREFIX="$WORK/prefix_proj/.conda" library-skills --check

# 2. Named conda environment
cd "$WORK/named_proj"
conda create --yes --quiet -n sedpack-conda-env python=3.11 pip
conda run -n sedpack-conda-env pip install --no-deps "$WHEEL"
CONDA_ENV_PATH=$(conda info --envs | awk '/^sedpack-conda-env / {print $NF}')
CONDA_PREFIX="$CONDA_ENV_PATH" library-skills install --skill sedpack --yes
test -L .agents/skills/sedpack
test -f .agents/skills/sedpack/SKILL.md
CONDA_PREFIX="$CONDA_ENV_PATH" library-skills --check
61 changes: 61 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# AGENTS.md

Sedpack is a scalable and efficient sharded dataset storage and streaming
library: a Python package (`src/sedpack/`), a Rust native extension (`rust/`,
built with PyO3 and `maturin` as `sedpack._sedpack_rs`), a user-facing LLM
agent skill (`src/sedpack/.agents/skills/sedpack/SKILL.md`), tutorials
(`docs/tutorials/`), a `pytest` suite (`tests/`), and an Astro Starlight
documentation website (`website/`).

If you are using `sedpack` in another project rather than developing `sedpack`
itself, read `src/sedpack/.agents/skills/sedpack/SKILL.md` for the high-level
`sedpack.io` API guide.

## Before Opening a Pull Request

- Read the contribution guidelines in `docs/contributing.md`.
- Keep each pull request focused on a single problem with a concise
description of what changed, why, and how it was tested. Avoid unrelated
refactoring, renames, formatting churn, version bumps, or `requirements.txt`
lock file churn unless explicitly requested.
- Every tracked `.py` and `.sh` file (except generated FlatBuffers files under
`src/sedpack/io/flatbuffer/shardfile/`) must include the Apache 2.0
copyright header. Verify with `./tools/check_copyright.sh`.

## Working in the Repository

- **Build and editable install**: Activate the virtual environment, install
dependencies (`pip install --editable ".[dev,tf]"`), and rebuild the Rust
extension after any change in `rust/` via `maturin develop --release`.
- **Unit tests**: Run `python -m pytest` from the repository root. When
TensorFlow is installed (`".[dev,tf]"`), no tests should be skipped.
- **Python formatting (`yapf`)**: Configured in `.style.yapf`. Check
formatting with `yapf --recursive --diff .` and format in-place with
`yapf --recursive --in-place .`.
- **Python type checking (`mypy`)**: Configured in `mypy.ini` (strict mode for
`sedpack.*`). Run `PYTHONPATH=./src mypy --no-color-output src docs`.
- **Python linting (`pylint`)**: Configured in `.pylintrc`. Run
`./tools/run_pylint.sh` (which runs `pylint *.py src docs`).
- **Rust checks**: From `rust/`, run `cargo +nightly fmt -- --check`,
`cargo +nightly clippy --all-targets`, and `cargo test`.
- **Markdown linting (`markdownlint`)**: All `*.md` and `**/*.md` files are
checked against `.markdownlint.json`:
- Wrap prose, list items, and block quotes at 80 characters (120 for
headings; tables and fenced code blocks are exempt).
- Use a hyphen followed by 3 spaces for unordered list items and a number
plus period followed by 2 spaces for ordered list items, with 4-space
indentation for nested lists and continuation lines.
- Always specify a language tag on fenced code blocks (`python`, `bash`,
`text`, `json`, `toml`).
- **Spell checking (`cspell`)**: Checked in CI against `cspell.json` and
`project-words.txt`. If a change introduces legitimate domain terms or
identifiers not recognized by `cspell`, add them in sorted order to
`project-words.txt`.
- **Generated files and skill synchronization**:
- Never hand-edit generated FlatBuffers bindings
(`src/sedpack/io/flatbuffer/shardfile/*.py` or
`rust/src/shard_generated.rs`); regenerate them from
`src/sedpack/io/flatbuffer/shard.fbs`.
- When modifying the public `sedpack.io` API, update
`src/sedpack/.agents/skills/sedpack/SKILL.md` so the bundled agent skill
stays accurate.
Loading
Loading