From d5bad5b8c24aee8ceaa79086cf37f007d28247a8 Mon Sep 17 00:00:00 2001 From: Andrey Golovanov Date: Sun, 25 Oct 2026 01:27:15 +0100 Subject: [PATCH] Fix DC-BB summaries and seed propagation and clarify documentation --- CHANGELOG.md | 13 + README.md | 226 ++++++---- dev/ARCHITECTURE.md | 136 +++--- dev/METRICS_VERIFICATION.md | 187 ++++----- experiments/dc-bb-interconnect/README.md | 104 +++-- .../dc-bb-interconnect/blueprints/blocks.yml | 18 - experiments/dc-bb-interconnect/run.py | 21 +- .../scenario.yml | 0 .../dc-bb-interconnect/workflows/analysis.yml | 14 +- metrics.md | 390 +++++++++++------- netlab/autoresearch/scenario_generator.py | 3 - .../dc-bb-autoresearch/program.md | 245 ++++++----- tests/autoresearch/test_generate_scenario.py | 32 ++ tests/data/README.md | 73 ++-- tests/test_dcbb_experiment.py | 122 ++++++ 15 files changed, 972 insertions(+), 612 deletions(-) delete mode 100644 experiments/dc-bb-interconnect/blueprints/blocks.yml rename experiments/dc-bb-interconnect/topologies/{dc4x9_bb4x4_bb4x4_dc4x9_two_to_one => dc4x9_bb2x4_bb2x4_dc4x9_two_to_one}/scenario.yml (100%) create mode 100644 tests/test_dcbb_experiment.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 69aae07..74c450b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## [Unreleased] +### Fixed + +- **Experiments**: DC-BB `summary.json` and `--compare` report `alpha_star` as the median over seeds, plus `alpha_star_by_seed`; zero shows as `0.00`. +- **Research**: Generated DC-BB workflow steps inherit the scenario seed, so `netlab autoresearch run --seed` changes failure sampling. +- **Research**: The DC-BB `program.md` example returns integer `g_abc1` and `g_xyz1`, which the parameter template accepts. + +### Changed + +- **BREAKING**: **Experiments**: The DC-BB workflow names its MSD step `msd_baseline`; read `steps.msd_baseline` in its results. +- **BREAKING**: **Experiments**: Rename the two-plane topology to `dc4x9_bb2x4_bb2x4_dc4x9_two_to_one`, matching its backbone grid. +- **Docs**: Explain setup, metric calculations and research workflows with self-contained examples and reproducible checks. +- **Internal**: Remove the unused DC-BB `blueprints/blocks.yml`. + ## [0.4.1] - 2026-10-25 ### Changed diff --git a/README.md b/README.md index 696ac72..e94289d 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,26 @@ # NetLab -NetLab builds and runs [NetGraph](https://github.com/networmix/NetGraph) scenarios -and compares bandwidth availability, latency stretch, capacity, and cost. -It also supports parameter sweeps and LLM-assisted topology experiments. +NetLab builds network scenarios with [TopoGen](https://github.com/networmix/TopoGen), +runs them with [NetGraph](https://github.com/networmix/NetGraph), and compares the +results. It reports how much traffic survives failures, how path costs change, +and how much the modeled equipment costs and consumes. + +You can run several configurations and random seeds in one batch. NetLab also +supports parameter searches and experiments proposed by a language model. ## Install -Requires Python 3.11+ and Git. NetGraph, TopoGen, and the other dependencies are -installed automatically; exact requirements are in [pyproject.toml](pyproject.toml). +Use Python 3.11 or later and Git. To install the command: ```bash -pip install git+https://github.com/networmix/NetLab +python -m pip install git+https://github.com/networmix/NetLab ``` -For the examples below, clone the repository and install the development environment: +This also installs NetGraph, TopoGen and the other dependencies. +[pyproject.toml](pyproject.toml) pins TopoGen to a release commit and lists the +supported NetGraph versions. + +To use the included examples, clone the repository and create its environment: ```bash git clone https://github.com/networmix/NetLab @@ -22,9 +29,11 @@ make dev source venv/bin/activate ``` -## Run scenarios +Run the commands below from the repository root. + +## Run an example -This example uses a small, checked-in graph and needs no external geographic data: +This example uses saved geographic routes, so no Census download is needed: ```bash netlab run topogen_configs_small/small_baseline.yml --seeds 11 12 \ @@ -33,56 +42,92 @@ netlab run topogen_configs_small/small_baseline.yml --seeds 11 12 \ netlab metrics scenarios/ ``` -Scenario YAML and results are saved under -`scenarios//__seed/`. Metrics, tables, and figures go into -`scenarios_metrics/`; start with `project.csv` and `BAC.png`. - -To generate geography from source data, omit `--graphs-dir`: +The first command builds a network for each seed and runs its NetGraph workflow. +A **seed** is the scenario's random seed. It drives TopoGen's random traffic +choices and NetGraph's failure sampling, so the same seed repeats the same +experiment. The second command calculates metrics and renders comparison figures. + +| Location | Contents | +| --- | --- | +| `scenarios//__seed/` | Scenario YAML, NetGraph results and task outcomes | +| `scenarios//_integrated_graph.json` | Geography generated when `--graphs-dir` is omitted | +| `scenarios/provenance.json` | Input hashes, settings, package versions and outcomes of the last batch | +| `scenarios_metrics/project.csv` | One summary row per configuration | +| `scenarios_metrics/BAC.png` | How often each delivered bandwidth was met or exceeded | +| `scenarios_metrics//seed/` | Metrics for one seed | + +To build without simulating, use `netlab build` with the same input arguments. +To generate geography yourself, omit `--graphs-dir` and first download the +[data required by TopoGen](https://github.com/networmix/TopoGen#readme). +The `data/` and `lib/` paths are relative to your working directory. From the +repository root, `lib/workflows.yml` replaces TopoGen's `design_analysis_brief` +workflow with NetLab's version, and `lib/failure_policies.yml` adds `mc_hard`. + +Saved geography must be a corridor `MultiGraph`: a graph that can keep several +routes between the same metro pair. Each file is named +`_integrated_graph.json`. The `topogen_configs/` directory contains larger +designs; `topogen_configs_small/` contains smaller versions for experimentation. + +## Control a batch + +| Option | Effect | +| --- | --- | +| `--build-jobs`, `--run-jobs` | Number of build and simulation tasks allowed to run at once | +| `--build-timeout` | Time limit for each geography or scenario build task | +| `--run-timeout` | Time limit for each simulation; defaults to 600 seconds | +| `--force-run` | Run simulations again even when cached results match | +| `--force` | Rerun simulations and regenerate geography, unless `--graphs-dir` supplies it | + +Time limits start when a task starts running. Scenario assembly runs every time. +NetLab reuses saved work only when a hash of its inputs still matches; a hash +identifies file contents, so any change to the inputs invalidates the cache. +Generated geography is reused while the configuration, data files and package +versions are unchanged. A simulation result is reused while the scenario, +package versions and the saved result file are unchanged. +Failed tasks make the command return a failure status; independent tasks continue. +Task outcomes are saved in `.run.json` files alongside the results. + +## Read and compare metrics ```bash -netlab run topogen_configs_small --seeds 11 12 -netlab build topogen_configs_small --seeds 11 12 # Build without simulating +netlab metrics scenarios/ --no-plots +netlab metrics scenarios/ --only small_baseline +netlab metrics scenarios/ --summary ``` -Source datasets and `lib/` are resolved from the working directory. Full-size -configurations are in `topogen_configs/`; reduced versions are in -`topogen_configs_small/`. Supplied graphs must be corridor MultiGraphs named -`_integrated_graph.json`. +`--no-plots` calculates tables without figures. `--only` takes a comma-separated +list of configuration names, such as `small_baseline,small_clos`. +`--summary` reads the saved tables and renders figures. +A calculation replaces the complete report directory, even with `--only`. +If calculation fails, the existing report is kept. -- `--build-jobs` and `--run-jobs` set process concurrency. -- `--build-timeout` sets a deadline per graph/build task; `--run-timeout` defaults - to 600 seconds per simulation. Deadlines start when a task begins executing. -- `--force-run` reruns simulations. `--force` also regenerates geography, unless - supplied through `--graphs-dir`. +The workflow must include these step names: -Scenario assembly runs every time. Cached results are reused only when scenario, -package, and result hashes match. Failed tasks return a nonzero CLI status; -independent tasks continue. Each run writes provenance, batch summaries, and -`.run.json` outcomes alongside its artifacts. +| Step name | NetGraph analysis | +| --- | --- | +| `msd_baseline` | `MaximumSupportedDemand`, which searches for a traffic multiplier | +| `tm_placement` | `TrafficMatrixPlacement` at the MSD multiplier, with flow details enabled | +| `network_statistics` | `NetworkStats`, which counts network objects | -## Analyze results +`--enable-maxflow` also needs a `node_to_node_capacity_matrix` step for pairwise +capacity analysis. See [metrics.md](metrics.md) for definitions, required fields, +worked examples and output files. Failure statistics describe the configured +experiment; they are not measurements of network uptime. + +To compare two configurations with matching seeds: ```bash -netlab metrics scenarios/ --no-plots -netlab metrics scenarios/ --only small_clos,small_dragonfly -netlab metrics scenarios/ --summary # Read saved tables and render figures netlab test scenarios/ small_baseline small_clos ``` -An analysis replaces the complete `scenarios_metrics/` report, including when -`--only` selects a subset. A failed analysis keeps the previous report. - -The metrics CLI requires workflow steps named `msd_baseline` -(`MaximumSupportedDemand`), `tm_placement` (`TrafficMatrixPlacement` with flow -details), and `network_statistics` (`NetworkStats`). `--enable-maxflow` adds -pairwise capacity analysis and requires `node_to_node_capacity_matrix`. - -See [metrics.md](metrics.md) for metric definitions, required fields, aggregation -rules, and output files. BAC describes the sampled failure experiment; latency -stretch describes delivered traffic relative to its baseline path cost. +Both configurations must have been analyzed. The statistical test needs at least +three matching finite seed results; the two-seed quick start is for running the +pipeline, not statistical inference. ## Python API +Save this as a script and point it to a NetGraph scenario: + ```python from pathlib import Path from netlab.simulation import run_simulation @@ -92,12 +137,14 @@ if __name__ == "__main__": print(outcome.status, outcome.error) ``` -`run_simulation` uses a local process queue and saves successful results. -Use `simulate(yaml_text)` for synchronous execution or `inspect_scenario(yaml_text)` -to validate and count model objects. Scripts using the process queue need the -`__main__` guard. See [the architecture notes](dev/ARCHITECTURE.md) for batch APIs. +`run_simulation` runs in a local worker process. A successful run saves +`scenario.results.json` next to the scenario; every run records its status in +`scenario.results.run.json`. The `__main__` guard keeps a spawned worker from +running the script again. Use `simulate(yaml_text)` to run in the calling +process, or `inspect_scenario(yaml_text)` to validate and count network objects. +See [architecture and batch APIs](dev/ARCHITECTURE.md). -Metrics can also be computed directly from a result dictionary: +Metric functions read a saved result dictionary: ```python import json @@ -113,66 +160,63 @@ print(compute_bac(results, step_name="tm_placement").auc_normalized) print(compute_latency_stretch(results).failures.get("p99")) ``` -## Autoresearch +They return the per-seed values described in [metrics.md](metrics.md). +`netlab metrics` also checks the result arithmetic before it computes them. -Create a parameterized project, review its templates, then run experiments: +## Automated experiments + +An autoresearch project gives a language model a task description +(`program.md`), the parameters it may change (`hypothesis_template.yml`) and the +metric to optimize (`objective.yml`). `init` copies a base scenario that has a +`MaximumSupportedDemand` step. Each `${{name}}` placeholder in it becomes a +parameter, and the objective maximizes `steps.msd_baseline.data.alpha_star`. +Review the generated files before a run: ```bash netlab autoresearch init --base-scenario scenario.yml --output project/ -netlab autoresearch run project/ --backend claude-cli --model sonnet +netlab autoresearch run project/ --backend mock ``` -Backends are `mock` (default), `claude-cli`, `codex-cli`, and `openai`. -`HypothesisManager` also accepts a free-text hypothesis: it generates and simulates -a scenario, computes metrics, and requests an interpretation and next experiment. -Cycle artifacts are saved under `/cycles/` for review. - -DC-BB layout studies have dedicated commands: - -```bash -netlab autoresearch structural-analysis --output layouts.json -netlab autoresearch sweep abc1 --output-dir results/ -netlab autoresearch cross-sweep --output-dir results/ -``` +The `mock` backend is for testing the workflow. To use an external model, select +`claude-cli`, `codex-cli` or `openai` and configure that backend's credentials. +CLI backends need their executable installed and signed in. +`HypothesisManager` can also turn a free-text hypothesis into a scenario, run it +and request an interpretation. Those cycle files are saved under `/cycles/`. -Research logs are tied to the inputs, seed, and installed dependencies. Use a new -output directory when these change, and one writer per project. Successful sweep -entries are skipped on resume; failed entries are retried. Scores rank higher as -better, and infeasible candidates cannot become best. +Run one writer per project; nothing locks the files. A project directory is tied +to its input files, seed and installed package versions. A sweep output directory +is tied to its sweep type, seed, iteration count and package versions. If any of +them changes, NetLab refuses to continue there; start a new directory. +Rerunning a sweep in its directory skips successful entries and retries failed +ones. Higher scores rank better; candidates that fail the configured constraints +cannot become best. -Study inputs: [DC-BB research](research_projects/dc-bb-autoresearch/program.md). -Standalone experiments: [DC-BB interconnect](experiments/dc-bb-interconnect/README.md). +For data-center-to-backbone designs, see the +[research project](research_projects/dc-bb-autoresearch/program.md) and +[included interconnect experiments](experiments/dc-bb-interconnect/README.md). ## Development ```bash -make check-ci # Formatting, lint, type checks, and tests -make check # Apply pre-commit fixes, then run tests and lint -make test # Tests with coverage -make lint # Formatting, lint, and type checks +make check-ci ``` -The tests include TopoGen → NetGraph → metrics runs using cached geography. -See [fixture regeneration](tests/data/README.md) and -[metric verification](dev/METRICS_VERIFICATION.md). +This checks formatting, lint, types and tests, including TopoGen → NetGraph → +metrics runs using saved geography. See [fixture regeneration](tests/data/README.md) +and [metric verification](dev/METRICS_VERIFICATION.md). -Superset setup creates a `venv`, installs `.[dev]`, and copies missing untracked -`.env` files from the root checkout. Its Run button executes `make check-ci`. -To run setup manually: +In Superset, `bash .superset/workspace.sh setup` creates a `venv`, installs the +development dependencies and copies missing untracked `.env` files from the root +checkout. The Run button executes `make check-ci`. -```bash -SUPERSET_ROOT_PATH=/path/to/NetLab bash .superset/workspace.sh setup -``` - -To test local upstream checkouts: +To test local checkouts of all three dependencies: ```bash -bash dev/check_ngraph_integration.sh ~/ws/NetGraph ~/ws/NetGraph-Core ~/ws/TopoGen +bash dev/check_ngraph_integration.sh /path/to/NetGraph /path/to/NetGraph-Core /path/to/TopoGen ``` -This builds Core and runs all checks in a temporary environment. Source revisions, -dependency versions, and results are saved under `build/ngraph-integration/`. - -## License +Replace the paths with your checkouts. The script builds Core and runs checks in +a temporary environment. It saves source revisions, dependency versions and +results under `build/ngraph-integration/`. -[MIT](LICENSE) +[Changelog](CHANGELOG.md) · [MIT license](LICENSE) diff --git a/dev/ARCHITECTURE.md b/dev/ARCHITECTURE.md index fcb4ce3..67f4d97 100644 --- a/dev/ARCHITECTURE.md +++ b/dev/ARCHITECTURE.md @@ -1,75 +1,97 @@ -# NetLab architecture +# How NetLab runs work -TopoGen builds geography and scenarios; NetGraph expands and simulates them. -NetLab calls their Python APIs, schedules work, saves results, and computes metrics. -CLI handlers pass arguments to these library services. +TopoGen creates geographic routes and network scenarios. NetGraph runs the +scenario's analysis steps. NetLab schedules those calls, saves their results +and turns the results into metrics, tables and figures. -## Execution and caching +## Worker processes and time limits -`TaskQueue` uses [Pebble](https://pebble.readthedocs.io/en/stable/) with a bounded -pool of spawned processes. Deadlines start when execution begins. A timed-out -worker is stopped and replaced, and independent jobs continue. No broker or worker -service is needed for these local batches. +`TaskQueue` runs tasks in a local pool of spawned processes using +[Pebble](https://pebble.readthedocs.io/en/stable/). The pool limits how many tasks +run at once. No separate worker service is needed. -`SimulationBatch` submits independent scenario files to a caller-managed queue. -Each simulation loads its scenario once. The parent publishes results after -success, and cache reuse requires matching scenario, package, and result hashes. -Editable packages include a source fingerprint. Failed builds invalidate earlier -scenario and result files. +A task's time limit starts when it begins executing. If it times out, its worker +is stopped and replaced. If a worker crashes, the pool replaces it too. +Other tasks can continue; failed tasks are reported without automatic retries. -`run_pipeline` generates one graph per master, builds each seed, and optionally -simulates it. Relative datasets and `lib/` resolve from the working directory. -Supplied geography must use TopoGen's corridor MultiGraph format. +`SimulationBatch` submits scenario files to a queue managed by its caller. +Each task loads a scenario once and executes its NetGraph workflow. The parent +process saves results only after success. + +## Pipeline and cache + +A master configuration is a TopoGen input file. `run_pipeline` generates or +loads one geographic graph per master configuration. +It builds a scenario for each requested seed and can then simulate each scenario. +Relative datasets and `lib/` definitions are read from the working directory. ```python from pathlib import Path -from netlab.pipeline import PipelineConfig, discover_configs, run_pipeline +from netlab.pipeline import PipelineConfig, run_pipeline if __name__ == "__main__": result = run_pipeline(PipelineConfig( - masters=discover_configs(Path("topogen_configs_small")), - seeds=[11, 12], output_dir=Path("scenarios"), - graphs_dir=Path("graphs"), build_jobs=2, run_jobs=2, + masters=[Path("topogen_configs_small/small_baseline.yml")], + seeds=[11, 12], + output_dir=Path("scenarios"), + graphs_dir=Path("tests/data/scenarios/small_baseline/small_baseline"), + build_jobs=2, + run_jobs=2, )) assert result.success, result.errors ``` -Failed tasks are reported without automatic retries. Research generation loops -can explicitly revise failed candidates using the recorded error. Subprocesses -remain only for external LLM CLI backends. +Run the example from the repository root. The supplied graph avoids a Census +download. Its filename prefix must match the master configuration. + +A generated graph is reused while the configuration, its data files and the +package versions are unchanged. Scenario assembly runs on each invocation; when +the rebuilt scenario differs from the saved one, its old results are removed. +A saved simulation is reused only when the scenario, package and result hashes +match. For editable packages, the package record also includes a fingerprint of +the source files, so source changes invalidate cached work. A failed build +removes the corresponding saved scenario and results rather than leaving them +available as valid outputs. + +## Metric calculation + +Metric functions read NetGraph result dictionaries. `metrics/analysis.py` +validates and analyzes one seed. `metrics/batch.py` groups seeds and creates a +complete report. That report replaces the destination only when analysis +succeeds. `metrics/reporting.py` reads saved metric files to produce summaries +and figures. + +A per-seed metric summarizes failure samples for one random seed. A project +metric usually takes the median of those seed summaries. Pooled curves instead +use all samples, so seeds with more samples have more weight. Definitions and +examples are in [metrics.md](../metrics.md). -## Where to make changes +## Research runs -| Area | Modules under `netlab/` | +Research runners use the same simulation service. A project defines inputs, +parameters, constraints and a score. Its log records input and dependency hashes +so resumed work uses the same experiment definition. Higher scores rank better; +a candidate that violates a constraint cannot become best. + +Parameter sweeps skip successful entries on resume and retry failed ones. +Generation loops can propose a revised candidate after a failure, using the +recorded error. External language-model CLI backends run as subprocesses. + +## Where to change the code + +Paths in the table are relative to `netlab/`. + +| Responsibility | Modules | | --- | --- | -| Task scheduling and results | `tasks.py`, `simulation.py`, `artifacts.py` | -| TopoGen batches | `topology.py`, `pipeline.py`, `cli.py` | -| Experiment inputs and runs | `scenario.py`, `experiment.py` | -| Tables and graph rendering | `comparison.py`, `visualize.py` | -| Research generation and analysis | `autoresearch/generation_loop.py`, `analysis_loop.py`, `hypothesis_manager.py` | -| Parameter searches | `autoresearch/hypothesis.py`, `runner.py`, `objective.py`, `sweep.py` | -| Research history and prompts | `autoresearch/experiment_log.py`, `memory.py`, `prompt.py`, `metrics_report.py` | -| LLM connections and research CLI | `autoresearch/backend.py`, `cli.py`; executable lookup in `runtime.py` | -| DC-BB topology model | `autoresearch/dcbb_config.py`, `dcbb_failures.py`, `scenario_generator.py`, `scenario_validation.py`, `structural_analysis.py` | -| Metric calculations | `metrics/bac.py`, `latency.py`, `msd.py`, `sps.py`, `costpower.py`, `iterops.py`, `matrixdump.py`; failure summaries in `metrics_failure.py` | -| Metric input and aggregation | `metrics/validation.py`, `common.py`, `analysis.py`, `aggregate.py`, `batch.py`, `seed_data.py` | -| Statistics and reports | `metrics/distributions.py`, `paired.py`, `comparisons.py`, `summary.py`, `reporting.py`, `plot_*.py` | - -Metric functions consume result dictionaries. `analysis.py` validates and analyzes -one seed; `batch.py` groups seeds and publishes a complete report. `reporting.py` -reads saved tables and metric files. Shared seed readers and statistical functions -keep table, plot, and comparison calculations consistent. - -Research runners use the same simulation service. Logs bind resumed work to input -and dependency fingerprints; feasible candidates are ranked by higher scores. -Only successful sweep entries are skipped on resume. - -## Checks - -Run `make check-ci` for formatting, lint, type checks, and tests. Execution tests -cover worker timeouts and crashes, cache invalidation, failed builds, and queued -TopoGen → NetGraph runs with cached geography. Research tests cover resume, -parameter validation, and ranking. Rendering tests check component bounds. - -For the independent metric experiments and their results, see -[METRICS_VERIFICATION.md](METRICS_VERIFICATION.md). +| Queue, simulation and saved outputs | `tasks.py`, `simulation.py`, `artifacts.py` | +| TopoGen builds and batch execution | `topology.py`, `pipeline.py`, `cli.py` | +| Experiment inputs | `scenario.py`, `experiment.py` | +| Metric calculation and validation | `metrics/` | +| Experiment failure summaries, comparisons and topology rendering | `metrics_failure.py`, `comparison.py`, `visualize.py` | +| Research loops, scoring and model connections | `autoresearch/` | +| Data-center-to-backbone model, failure modes and sweeps | `autoresearch/dcbb_config.py`, `scenario_generator.py`, `dcbb_failures.py`, `structural_analysis.py`, `sweep.py` | + +Run `make check-ci` after changes. Tests cover worker timeouts and crashes, +cache invalidation, failed builds, metric arithmetic and research resume/ranking. +The [metric verification guide](METRICS_VERIFICATION.md) explains how to run +small network experiments with independently calculated answers. diff --git a/dev/METRICS_VERIFICATION.md b/dev/METRICS_VERIFICATION.md index 2c32da8..7e44e3d 100644 --- a/dev/METRICS_VERIFICATION.md +++ b/dev/METRICS_VERIFICATION.md @@ -1,108 +1,93 @@ -# Metrics verification +# Check metric calculations -The numerical audit covers the current NetLab metric pipeline and its research -consumers. Evidence and rendered figures live in `build/metrics-verification/`. -The runnable experiment is `dev/verify_metrics.py`; regression tests are in -`tests/metrics/test_numerical_verification.py` and `test_native_verification.py`. +`dev/verify_metrics.py` creates small networks, runs NetGraph and compares NetLab's +metrics with answers calculated separately. The networks are small enough to +check by hand or with a separate maximum-flow calculation. -## Reproduce +This page explains how to run those checks and interpret them. The metric +reference is [metrics.md](../metrics.md). -```sh +## Run the checks + +From the repository root, install `.[dev]` in the project's virtual environment. +Then run: + +```bash venv/bin/python dev/verify_metrics.py --plots venv/bin/python -m pytest tests/metrics tests/test_current_ngraph_integration.py tests/test_mini_dcbb_verification.py --no-cov -make check-ci ``` -The experiment performs 38 configured native runs: 16 exhaustive failure masks, -six successful model variants, one expected disconnected-MSD rejection, and -15 scenario/seed runs. It also constructs a 136-sample weighted population from -the native outcomes. Those 136 observations are reweighted native results, not -136 additional engine executions. - -## Checks - -| Area | Evidence | Result | -| --- | --- | --- | -| BW@p / SPS@p | Rational counting oracle at 13 sample lengths from 1 to 10,001; exact 90/95/99/99.9/99.99% boundaries and replication | Passed after rank correction | -| Empirical survival | 100 random tied integer samples, direct threshold counts; cross-seed plots and saved curves | Passed | -| Latency | 100 independently volume-expanded distributions; distinct devices, priorities, zero delivery, cost-unit rescaling, missing/partial references | Passed after endpoint and tolerance fixes | -| MSD | Native diamond capacity, pairwise volume splitting, combine, bidirectional traffic, bounded search | Passed; no feasible positive alpha produces an engine error | -| Pair matrices / SPS | All 16 edge-failure subsets vs NetworkX maximum flow and two-path arithmetic | Passed; mismatched group labels rejected | -| Cost / power | Four 100-dollar/10-watt routers and eight 2-dollar/1-watt optics; global and per-metro totals 416 dollars/48 watts; independent divisions | Passed; missing is distinct from zero | -| Iterations | Occurrence expansion, 136 weighted failures, metadata reconciliation, nonnegative time | Passed; timing is amortized, not a throughput benchmark | -| Seed aggregation | Three capacity configurations × five real seeds, 9/19/29/39/49 failures per seed; manual medians, ratios, deltas, pooled counts and IQR | Passed across all non-timing project metrics | -| Inference | SciPy paired t and CIs over scales 1e-150..1e150; invariance at 1e-200 and 1e200; constant decimals; missing pairs; hand-calculated Holm | Passed after stable scaling and validation | -| Input contract | Mutation of native failure/baseline demand, placed, dropped, costs, summaries, alpha, occurrence counts, duration and endpoints | Passed; invalid numbers and missing required fields rejected | -| Artifacts | Reduced-input rerun, MaxFlow removal, late-seed failure, source mutation during analysis, filename seed mismatch, strict JSON | Passed; complete-tree publication and retained prior report on failure | -| Plots | Inclusive pre-steps, exact observation thresholds, common baseline, seed IQR, adjusted significance, visual inspection | Passed; single/pooled BAC, delta, latency and heatmap renders inspected | -| Research consumers | Shared BAC computation, explicit availability field, consistent lower percentiles, unrounded stored values | Updated and covered by the full research suite | - -## Defects found and fixed - -The first new numerical suite had 18 failures before correction. Evidence is in -`before.json` and `before-tests.log`; later regression logs are saved separately. - -| Finding | Reproducer | Correct behavior | -| --- | --- | --- | -| Incorrect availability rank | `[0,...,9]` returned BW@90=0 | BW@90=1: exactly 9/10 samples meet it | -| Endpoint aliasing | Unchanged device paths of cost 1 and 10 produced p99 stretch 10 | Each device has its own reference; p99=1 | -| Unmatched MaxFlow selectors | Native regex group labels produced SPS=0 on a fully connected network | Exact endpoint coverage is required; incompatible groups raise a diagnostic | -| Missing cost represented as free | No CostPower step produced zero unit cost | Unavailable values serialize as null | -| Independent sample positions aligned | Permuting one seed changed positional median/IQR | Positional reduction removed; compare scalar seed summaries and survival curves | -| Stale/mixed reports | Removed seed and MaxFlow files survived; a failing later seed partially rewrote output | Entire successful report replaces prior output; failure preserves prior bytes | -| Undefined baseline normalized to one | Missing/zero denominators in the baseline row became 1 | They remain unavailable | -| Extreme-scale inference | Nonconstant differences at 1e-200 became deterministic; at 1e200 variance overflowed | Scale before variance; preserve p/t and rescale CI | -| Best-path tolerance depended on units | Doubling path cost at 1e-12 scale still counted as best path | Relative tolerance, no arbitrary absolute floor | -| Plot/report inconsistency | Single-seed curve used post-steps; baseline choice and clipping differed | Inclusive pre-steps, common selector, uncapped delivery curves | -| Partial latency coverage hidden | Delivered traffic without a baseline reference vanished from the denominator | `reference_coverage` explicitly reports the included delivered fraction | - -## Interpretation - -- BAC deliberately samples one baseline plus occurrence-weighted failures. - SPS, pair matrices, and failure latency use failures only. These summaries - describe the configured experiment; they do not measure uptime. -- BW@p uses sorted zero-based rank `n - ceil(n*p/100)`, with exact decimal count - boundaries. Descriptive high percentiles retain NumPy's `lower` estimator; - they are not availability guarantees and need not be replication invariant. -- A seed median uses equal seed weight; pooling uses equal observation weight. - Seeds with more observations therefore contribute more to pooled results. -- MaxFlow capacities are independent per pair; their sum does not establish - simultaneous multicommodity feasibility. Group-to-device reassignment cannot - be inferred safely from labels alone and is explicitly unsupported. -- Latency is conditional on delivered volume with a positive baseline reference. - Outages retain missing entries; reference coverage distinguishes absent path - details and partially referenceable delivery. Inspect BAC alongside latency. -- Zero baseline delivery has defined absolute bandwidth and undefined ratios. - NetGraph's MSD workflow rejects a network with no positive feasible multiplier. -- Pairwise demand expansion divides volume across matched pairs/groups, as - confirmed by source inspection and the native experiment. -- Statistical tests require three finite matched seeds and the usual paired-test - assumptions. Constant-sample p=0/1 is a computational convention, not certainty - about the population. Finite samples cannot establish multi-nine uptime. -- The audit covers arithmetic and integration for the listed models. Recorded - durations depend on the environment and are not benchmark results. - -Sources for definitions: [NumPy quantile methods](https://numpy.org/doc/stable/reference/generated/numpy.quantile.html), -[SciPy paired t-test and confidence interval](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_rel.html). - -## Validation results - -Results from the metric verification run. - -- Independent native experiment: passed, including 435 failure trials across the - 15 multi-seed placement studies and an independently calculated reference table. -- Numerical/native targeted checks: passed; complete metrics/current-engine suite - passed 138 tests after restoring strict demand selectors. -- `make check-ci`: **516 passed in 153.43 s**, overall coverage 80%; Ruff passes; - Pyright has zero errors and three missing-stub warnings (SciPy/CairoSVG). -- Wheel and sdist build; `twine check` and `pip check` pass. -- The rebuilt wheel was force-installed into a separate environment with its own - resolved dependencies. The import path was asserted to be its `site-packages`. - A real mini-DCBB run reproduced AUC=6/11; the full 15-study report matched the - source-tree report across every numeric project column at rtol=1e-12. -- The eight saved geographic fixture analyses were regenerated. Released - changelog history is unchanged byte-for-byte. No commit or push was performed. - -Key evidence files: `experiments.log`, `native-summary.json`, -`multiseed-expected.json`, `multiseed_metrics/project.csv`, `check-ci.log`, -`build.log`, `twine.log`, `pip-check.log`, and `wheel-smoke.log`. +The first command writes to `build/metrics-verification/`. Use `--output PATH` +to choose another directory. `--plots` also renders report figures. The output +contains the NetGraph results and the independently calculated answers; the input +networks are defined in `dev/verify_metrics.py`. A failed comparison raises an +error and makes the command fail. The second +command runs the metric regression and integration tests. For formatting, lint, +type checks and the complete test suite, run `make check-ci`. + +## A network you can check by hand + +The verification network has two routes between the same endpoints: + +| Route | Capacity | Total cost | +| --- | ---: | ---: | +| Through node `u` | 5 | 2 | +| Through node `v` | 3 | 4 | + +With both routes available, it carries 8 units. If a link on the first route +fails, only 3 units can be delivered. The experiment includes one baseline and +one failure sample, so its bandwidth ratios are `1` and `3/8`. Their mean, +the capped BAC score, is `(1 + 3/8) / 2 = 0.6875`. + +The failure's delivered traffic uses the route of cost 4. The cheapest baseline +route costs 2, so its latency stretch is `4/2 = 2`. This ratio measures path cost, +not elapsed network delay. + +There are four routers at 100 dollars and 10 watts each, plus eight optics at +2 dollars and 1 watt each. The totals are `4*100 + 8*2 = 416` dollars and +`4*10 + 8*1 = 48` watts. MSD accepts the 8 units unchanged (`alpha_star = 1`), +so `USD_per_Gbit_offered` is `416 / (1 * 8) = 52`. + +The script repeats the experiment for all 16 subsets of its four physical links. +It checks capacities against both NetworkX maximum flow and the two-route sum. +It also checks weighted samples, placement variants and several random seeds. + +## What the checks cover + +| Area | Independent check | +| --- | --- | +| Bandwidth at a stated availability | Count samples that meet each threshold, including exact decimal boundaries and repeated samples | +| Availability curves | Count observations at or above each distinct delivered-bandwidth value | +| Latency stretch | Expand delivered-volume distributions and use a separate baseline cost for each endpoint pair | +| Pair capacity and SPS | Compare each failed graph with maximum flow and the two-route arithmetic | +| Cost and power | Add device totals and divide by independently calculated bandwidth | +| Seed summaries | Calculate medians, ratios, differences and pooled counts separately | +| Statistical comparisons | Compare paired tests with SciPy, including very small/large scales and constant differences | +| Invalid inputs | Change demand, delivery, costs, occurrence counts and endpoint fields; require a diagnostic | +| Report replacement | Reduce inputs or force a late failure; check that output is complete and failure keeps the saved report | + +Regression cases are in `tests/metrics/test_numerical_verification.py`, +`tests/metrics/test_native_verification.py` and the other files under `tests/metrics/`. +The CLI tests also check tables and report replacement. + +## Inspect the output + +| File or directory | Contents | +| --- | --- | +| `diamond-mask-*.json` | NetGraph results for each failed-link subset | +| `native-summary.json` | Expected and measured values for the small network | +| `variant-*.json` | Results for the additional model cases | +| `disconnected-msd.json` | The expected rejection when no positive demand multiplier is feasible | +| `multiseed/` | NetGraph results for three capacity variants, each with five seeds | +| `multiseed-expected.json` | Independently calculated summaries across seeds | +| `multiseed_metrics/` | Generated tables and optional figures for those results | + +Read bandwidth and latency together. A total outage has zero delivered bandwidth +and no meaningful delivered-path latency. Pairwise maximum flow treats each pair +independently; its sum does not prove that all pairs can carry traffic at once. +These experiments check arithmetic and integration, not real-world uptime or +execution speed. + +For the statistical methods, see +[NumPy quantiles](https://numpy.org/doc/stable/reference/generated/numpy.quantile.html) +and [SciPy's paired t-test](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_rel.html). diff --git a/experiments/dc-bb-interconnect/README.md b/experiments/dc-bb-interconnect/README.md index 2fd2ac2..58bbd5e 100644 --- a/experiments/dc-bb-interconnect/README.md +++ b/experiments/dc-bb-interconnect/README.md @@ -1,52 +1,96 @@ -# DC-BB interconnect experiments +# Compare data-center-to-backbone connections -Compare connections between data-center (DC) and backbone (BB) devices at two -sites. DC rows represent spine rows; BB rows represent planes. Matching backbone -planes connect the sites. Backbone planes have no direct links to one another -within a site. +These experiments compare ways to connect data-center (DC) devices to backbone +(BB) devices at two sites. DC rows represent spine rows. BB rows represent +backbone planes: groups of devices that can share a failure. Matching planes +connect the sites; there are no direct links between different planes at one site. -Topology directories are named -`dc{rows}x{devices}_bb{planes}x{devices}_bb{planes}x{devices}_dc{rows}x{devices}_{pattern}`, -ordered DC-A, BB-A, BB-B, DC-B. Each directory's `scenario.yml` defines its -connections and capacities. Use `--list` to see the available configurations. +Each topology directory contains a `scenario.yml` with its devices, connections +and link capacities. Directory names list the grids in this order: DC-A, BB-A, +BB-B, DC-B. `dc4x9` means 4 DC rows of 9 devices, and `bb4x4` means 4 backbone +planes of 4 devices. For example, `dc4x9_bb4x4_bb4x4_dc4x9_one_to_one` has 36 DC +and 16 BB devices at each site. The suffix describes how DC rows connect to +backbone planes. -## Run +## Run an included topology -From the repository root, with the development environment installed: +From the repository root, create the development environment with `make dev`. +Then run: ```bash source venv/bin/activate cd experiments/dc-bb-interconnect python run.py --list - -# Run an included topology with three seeds python run.py dc4x9_bb4x4_bb4x4_dc4x9_one_to_one --seeds 42 43 44 +``` + +The runner adds shared inputs to the topology's `scenario.yml`: hardware +components from `../_shared/components.yml`, failure policies from `policies/`, +demands from `demands/` and the workflow that the topology names from +`workflows/`. Each topology defines its own blueprints. The seed becomes the +scenario seed, which selects the sampled failures. A range such as `--seeds 42:50` +includes 42 through 49. + +The `failure_analysis` workflow first runs `msd_baseline`, a +`MaximumSupportedDemand` search for the largest multiplier of the inter-site +traffic the network can carry. Six placement steps then place that scaled +traffic under 1,000 sampled failures each. Every sample fails one randomly chosen +element of the step's type: a DC-to-BB link (`tm_dc_bb_link`), an inter-site +BB link (`tm_bb_bb_link`), a DC device (`tm_dc_node`), a BB device +(`tm_bb_node`), all inter-site links of one plane (`tm_plane`) or all devices of +one DC row at one site (`tm_dc_row`). -# Seed range: 42 through 49 -python run.py dc4x9_bb4x4_bb4x4_dc4x9_one_to_one --seeds 42:50 +To write the merged scenario without running the analysis: -# Write the merged scenario without simulating +```bash python run.py dc4x9_bb4x4_bb4x4_dc4x9_one_to_one --dry-run +``` + +To analyze saved results, compare topologies or draw the network: -# Analyze existing results and compare topologies +```bash python run.py --metrics python run.py --compare - -# Render the topology as SVG; --split stacks disconnected components python run.py dc4x9_bb4x4_bb4x4_dc4x9_one_to_one --visualize --split ``` -Add `--force` to rerun cached simulations. Results are written to -`results/{topology}/`, with raw NetGraph outputs in `{topology}__seed{N}/` and -aggregated failure statistics in `summary.json`. +`--visualize` draws a saved result as `results//topology.svg`. +`--split` also writes `topology_split.svg`, with disconnected components stacked +so they can be inspected separately. `--force` reruns cached simulations. + +## Find the results + +Results are saved under `results//` relative to this experiment +directory. Each `__seed/` subdirectory holds the merged scenario and +its NetGraph results. `summary.json` combines the seeds: + +- `alpha_star_by_seed` lists each seed's MSD multiplier, or `null` when a result + has none; `alpha_star` is the median of the available values, zero included. +- `failure_analysis` gives, for each placement step, the minimum, mean and + maximum fraction of the requested traffic that was delivered, over the failure + samples of all seeds. +- `worst_failures` names the steps whose minimum fraction is lowest. + +`python run.py --compare` prints these summaries side by side and saves +`results/comparison.json`. `Alpha* %Max` is each topology's `alpha_star` as a +percentage of the largest one in the comparison. + +Read the fractions together with `alpha_star`. Each topology is loaded to its +own MSD multiplier, so a topology with a lower `alpha_star` can deliver a larger +fraction of a smaller load and still carry less traffic. See +[the metric definitions](../../metrics.md). -## Included patterns +## Connection patterns -- `one_to_one`: each DC row attaches to its corresponding BB plane. -- `full_mesh`: each DC row attaches to every BB plane. -- `balanced_sparse`, `balanced_dense`, and other named variants: partial - connections defined in their scenario files. +| Pattern | Connections | +| --- | --- | +| `one_to_one` | Each DC row attaches to the corresponding BB plane | +| `full_mesh` | Each DC row attaches to every BB plane | +| `two_to_one` | Two DC rows share one BB plane | +| `balanced_sparse`, `balanced_dense` and named variants | Selected connections listed in that topology's scenario | -The small configurations use 4×9 DC grids and four BB planes per site, except -`two_to_one`, which has two planes. The larger `one_to_one` case uses a 16×36 DC -grid and 16×4 backbone per site. +Most small examples use 4×9 DC grids and four backbone planes of four devices at +each site; `dc4x9_bb2x4_bb2x4_dc4x9_two_to_one` has two planes. The larger +`one_to_one` example uses a 16×36 DC grid and 16 planes of four devices at each +site. Use `--list` and inspect `scenario.yml` to select the configuration you want +to compare. diff --git a/experiments/dc-bb-interconnect/blueprints/blocks.yml b/experiments/dc-bb-interconnect/blueprints/blocks.yml deleted file mode 100644 index 47a8a39..0000000 --- a/experiments/dc-bb-interconnect/blueprints/blocks.yml +++ /dev/null @@ -1,18 +0,0 @@ -# Shared blueprints for DC-BB interconnect experiments - -blueprints: - # DC row blueprint - instantiated multiple times per site - # Override node count via params when instantiating - DC_Row: - nodes: - node: - count: 1 - template: "{n}" - - # BB plane blueprint - instantiated multiple times per site - # Override node count via params when instantiating - BB_Plane: - nodes: - node: - count: 1 - template: "{n}" diff --git a/experiments/dc-bb-interconnect/run.py b/experiments/dc-bb-interconnect/run.py index 5c5840f..db61ce0 100755 --- a/experiments/dc-bb-interconnect/run.py +++ b/experiments/dc-bb-interconnect/run.py @@ -72,6 +72,7 @@ def _compute_scenario_metrics( "seeds": [], "network": {}, "alpha_star": None, + "alpha_star_by_seed": {}, "failure_analysis": {}, } @@ -90,8 +91,7 @@ def _compute_scenario_metrics( self._check_bb_distribution(results, summary) alpha = extract_alpha_star(results) - if alpha: - summary["alpha_star"] = alpha + summary["alpha_star_by_seed"][str(seed)] = alpha analysis = analyze_results(results) for step_name, stats in analysis.failure_stats.items(): @@ -103,6 +103,13 @@ def _compute_scenario_metrics( summary["failure_analysis"][step_name]["iterations"] += stats.iterations summary["failure_analysis"][step_name]["ratios"].extend(stats.ratios) + alphas = [ + alpha + for alpha in summary["alpha_star_by_seed"].values() + if alpha is not None + ] + summary["alpha_star"] = statistics.median(alphas) if alphas else None + for step_name, data in summary["failure_analysis"].items(): ratios = data["ratios"] summary["failure_analysis"][step_name] = { @@ -204,17 +211,19 @@ def generate_comparison(self, scenarios: Optional[List[str]] = None) -> None: builder.add_row( "Alpha*", lambda s: s.get("alpha_star"), - formatter=lambda v: f"{v:.2f}" if v else "?", + formatter=lambda v: f"{v:.2f}", ) alphas = [ - s.get("alpha_star") for s in summaries.values() if s.get("alpha_star") + s["alpha_star"] + for s in summaries.values() + if s.get("alpha_star") is not None ] - max_alpha = max(alphas) if alphas else 1 + max_alpha = max(alphas, default=0) builder.add_row( "Alpha* %Max", lambda s: s.get("alpha_star"), - formatter=lambda v: f"{v / max_alpha * 100:.1f}%" if v else "?", + formatter=lambda v: f"{v / max_alpha * 100:.1f}%" if max_alpha > 0 else "?", ) builder.add_row( diff --git a/experiments/dc-bb-interconnect/topologies/dc4x9_bb4x4_bb4x4_dc4x9_two_to_one/scenario.yml b/experiments/dc-bb-interconnect/topologies/dc4x9_bb2x4_bb2x4_dc4x9_two_to_one/scenario.yml similarity index 100% rename from experiments/dc-bb-interconnect/topologies/dc4x9_bb4x4_bb4x4_dc4x9_two_to_one/scenario.yml rename to experiments/dc-bb-interconnect/topologies/dc4x9_bb2x4_bb2x4_dc4x9_two_to_one/scenario.yml diff --git a/experiments/dc-bb-interconnect/workflows/analysis.yml b/experiments/dc-bb-interconnect/workflows/analysis.yml index 5e1f58a..d43428e 100644 --- a/experiments/dc-bb-interconnect/workflows/analysis.yml +++ b/experiments/dc-bb-interconnect/workflows/analysis.yml @@ -9,7 +9,7 @@ workflows: - type: NetworkStats name: network_statistics - type: MaximumSupportedDemand - name: msd + name: msd_baseline demand_set: inter_site alpha_start: 1.0 resolution: 0.01 # MSD search increment. @@ -19,7 +19,7 @@ workflows: demand_set: inter_site failure_policy: single_dc_bb_link iterations: 1000 - alpha_from_step: msd + alpha_from_step: msd_baseline alpha_from_field: data.alpha_star # BB-BB link failure (inter-site) - type: TrafficMatrixPlacement @@ -27,7 +27,7 @@ workflows: demand_set: inter_site failure_policy: single_bb_bb_link iterations: 1000 - alpha_from_step: msd + alpha_from_step: msd_baseline alpha_from_field: data.alpha_star # DC node failure - type: TrafficMatrixPlacement @@ -35,7 +35,7 @@ workflows: demand_set: inter_site failure_policy: single_dc_node iterations: 1000 - alpha_from_step: msd + alpha_from_step: msd_baseline alpha_from_field: data.alpha_star # BB node failure - type: TrafficMatrixPlacement @@ -43,7 +43,7 @@ workflows: demand_set: inter_site failure_policy: single_bb_node iterations: 1000 - alpha_from_step: msd + alpha_from_step: msd_baseline alpha_from_field: data.alpha_star # Plane failure (all inter-site links of one plane) - type: TrafficMatrixPlacement @@ -51,7 +51,7 @@ workflows: demand_set: inter_site failure_policy: single_plane iterations: 1000 - alpha_from_step: msd + alpha_from_step: msd_baseline alpha_from_field: data.alpha_star # DC row failure (all nodes in one DC row at one site) - type: TrafficMatrixPlacement @@ -59,5 +59,5 @@ workflows: demand_set: inter_site failure_policy: single_dc_row iterations: 1000 - alpha_from_step: msd + alpha_from_step: msd_baseline alpha_from_field: data.alpha_star diff --git a/metrics.md b/metrics.md index 57eec7b..69f970f 100644 --- a/metrics.md +++ b/metrics.md @@ -1,163 +1,233 @@ -# Metrics and output files - -`netlab metrics` writes per-seed results, scenario summaries, and project tables. -Project metrics are medians across seeds unless stated otherwise. Failure samples -are weighted by each pattern's `occurrence_count`; BAC also includes the baseline -as one sample. Availability describes the configured failure experiment, not -measured network uptime. - -MSD demand records use `source`, `target`, and `volume`; placement flow records -use `source`, `destination`, and `demand`. Each failure pattern must have a positive -integer `occurrence_count`. - -## Capacity and bandwidth - -| Column | Definition | -|--------|------------| -| `scenario` | Scenario directory name. | -| `seeds` | Number of seed summaries included. | -| `node_count`, `link_count` | Median topology counts across seeds. | -| `alpha_star` | Demand multiplier returned by the MSD search, subject to its bounds, resolution, and placement settings. | -| `bac_auc` | Mean of `min(delivered / baseline_delivered, 1)` across samples. | -| `bw_p99`, `bw_p999` | Greatest observed bandwidth met or exceeded by at least 99% or 99.9% of samples, divided by baseline delivery. | - -BAC uses baseline delivered bandwidth as its normalization value, called -`offered` in the API. For sorted samples and availability probability `p`, -`BW@p` selects zero-based rank `n - ceil(n*p/100)`. Decimal probabilities use -exact count boundaries: for `[0,...,9]`, `BW@90 = 1`, since nine of ten values -are at least 1. This inverse-survival threshold differs from a descriptive -percentile. `quantiles_abs`, `quantiles_pct`, SPS `tails`, pair matrices and -sweep percentile arrays use the NumPy `lower` estimator, rank `floor((n-1)*q)`. -Raw ratios and curves are uncapped; AUC is capped at 1. - -BAC, pair matrices, and SPS sum priority classes with the same exact source and -destination. Fully disconnected iterations retain their zero bandwidth samples. -Survival curves use exact inclusive thresholds, combining ties without linear -interpolation. - -Optional structural pair survivability (SPS) uses independently computed MaxFlow -capacities: `sum(min(pair_capacity, pair_demand)) / sum(pair_demand)` per failure -iteration. It does not measure simultaneous traffic placement. Enable it with -`--enable-maxflow` and a `node_to_node_capacity_matrix` step. Its baseline must -cover the exact placement source/destination pairs. For physical pairwise -traffic, use full-name capture groups in MaxFlow selectors, such as `^(A)$` -and `^(B)$`; selectors without captures label groups with the regex itself. -Group or virtual endpoint labels that do not match placement pairs raise an error. -`SPS@p` uses the same inverse-survival threshold as BAC, over failure samples only. +# Read NetLab metrics + +`netlab metrics` turns NetGraph results into per-seed metric files, scenario +summaries and project tables. A seed is one repeatable set of random choices. +Every numeric project column except `seeds` is the median of that metric's +per-seed values. + +A failure pattern can stand for several sampled iterations. Its positive integer +`occurrence_count` is its weight: a pattern with count 10 contributes ten samples. +Bandwidth availability describes these configured samples, not measured uptime. + +## Required results + +The metrics CLI expects workflow steps named `msd_baseline` +(`MaximumSupportedDemand`), `tm_placement` (`TrafficMatrixPlacement`, with flow +details), and `network_statistics` (`NetworkStats`). Optional pair-capacity +analysis uses `node_to_node_capacity_matrix`; equipment totals use `cost_power`. + +`tm_placement` must place the traffic scaled by the MSD result. Set +`alpha_from_step: msd_baseline` and `alpha_from_field: data.alpha_star` on that +step. NetLab checks that the baseline and every failure sample request exactly +`alpha_star` times the total MSD base demand, so all placement metrics describe +the network at its supported load. + +MSD demand records use `source`, `target` and `volume`. Placement flow records +use `source`, `destination`, `demand` and `placed`, the delivered part. Keep full +endpoint names: different devices need separate bandwidth and path-cost references. + +## Capacity and delivered bandwidth + +**Maximum supported demand (MSD)** searches for a multiplier of the configured +traffic. Its result, `alpha_star`, depends on the search bounds, resolution and +placement policy. For example, `alpha_star=2` means the search accepted twice the +base traffic. It is not a claim about other traffic distributions. + +**Bandwidth availability (BAC)** compares delivered traffic in each sample with +the amount delivered in the no-failure baseline. The AUC score is the area under +the availability curve, calculated here as the mean capped bandwidth ratio. +BAC includes that baseline as one sample, followed by the occurrence-weighted +failure samples. + +| Column | Meaning | +| --- | --- | +| `scenario` | Scenario directory name | +| `seeds` | Number of seed summaries included | +| `node_count`, `link_count` | Median topology counts across seeds | +| `alpha_star` | Demand multiplier returned by MSD | +| `bac_auc` | Mean of `min(delivered / baseline_delivered, 1)` across BAC samples | +| `bw_p90`, `bw_p95`, `bw_p99`, `bw_p999` | Bandwidth met or exceeded by at least 90%, 95%, 99% or 99.9% of BAC samples, divided by baseline delivery | + +For example, suppose the baseline delivers 10 Gbps and two failure samples deliver +10 and 5 Gbps. BAC uses ratios `[1, 1, 0.5]`, so `bac_auc = 2.5/3`. +Raw bandwidth ratios and curves can exceed 1; the AUC score caps each ratio at 1. +The API calls the baseline delivered amount `offered`. Project bandwidth columns +are ratios: `0.5` means 50% of baseline delivery. BAC plots label that value as 50%. + +### Bandwidth at an availability level + +`BW@p` is the greatest observed bandwidth that at least `p` percent of samples +meet or exceed. For sorted samples, its zero-based index is +`n - ceil(n*p/100)`. Decimal probabilities use exact count boundaries. + +For `[0,1,2,3,4,5,6,7,8,9]`, `BW@90 = 1`: nine of the ten samples are at least 1. +This is different from an ordinary descriptive percentile. The fields +`quantiles_abs`, `quantiles_pct`, SPS `tails`, pair matrices and sweep percentile +arrays use NumPy's `lower` estimator: index `floor((n-1)*q)` for quantile `q`. + +BAC, pair matrices and SPS combine priority classes with the same exact source +and destination. A total outage keeps its zero-bandwidth sample. Availability +curves count values at or above each threshold, including ties, without +interpolating between observations. + +### Independent pair capacity + +**Structural pair survivability (SPS)** asks how much of each pair's demand its +independently calculated maximum flow could carry. For each failure sample: + +`SPS = sum(min(pair_capacity, pair_demand)) / sum(pair_demand)` + +For two pairs with demand 10 each and capacities 10 and 5, SPS is `15/20 = 0.75`. +Each maximum flow is solved independently; this does not prove that both pairs +can carry those amounts at the same time. + +Enable it with `--enable-maxflow`. The `node_to_node_capacity_matrix` step must +cover every placement endpoint pair under its exact names. To label physical +endpoints with their full names, use capture groups such as `^(A)$` and `^(B)$` +in the MaxFlow selectors. Without captures, NetGraph labels a selected group with the selector +text itself. Labels that do not match placement pairs cause an error. +`SPS@p` uses the same availability-threshold rule as BAC, over failure samples only. ## Latency stretch -Each exact source/destination pair has its own reference: the minimum -cost carrying nonzero volume in its baseline `cost_distribution`. Stretch is -path cost divided by that reference. Percentiles and shares are weighted by -delivered volume. Failure summaries take the median across iterations, then -project tables take the median across seeds. - -| Column | Definition | -|--------|------------| -| `lat_base_p50` | Baseline median stretch. | -| `lat_fail_p99` | Median of the failure iterations' volume-weighted p99 stretches. | -| `lat_TD99` | Failure p99 summary divided by baseline p99. | -| `lat_SLO_1_2_drop` | Baseline share with stretch ≤1.2 minus the median failure share. | -| `lat_best_path_drop` | Baseline share at the reference cost minus the median failure share. | -| `lat_WES_delta` | Median failure weighted excess stretch minus baseline weighted excess stretch. | - -Weighted excess stretch is the volume-weighted mean of `max(stretch - 1, 0)`. -Latency metrics describe delivered traffic with a positive baseline cost reference; -dropped traffic contributes no path cost. `reference_coverage` in per-seed JSON -records the included fraction of delivered volume. It is zero for missing path -details and undefined when nothing is delivered. Fully disconnected iterations -retain missing entries and are excluded from latency medians. Read latency alongside BAC and reference coverage. -`lat_fail_p99` describes a typical iteration's tail, not the worst failure. - -## Iterations, timing, and cost - -| Column | Definition | -|--------|------------| -| `iters_fail` | Sum of failure-pattern occurrence counts. | -| `iters_total` | Failure iterations plus one baseline. | -| `unique_patterns` | Number of distinct failure records. | -| `tm_duration_total_sec` | Recorded `tm_placement` duration in seconds. | -| `tm_duration_per_iter_sec` | Total duration divided by total iterations. | -| `capex_total` | Total capital cost reported by the `cost_power` step. | -| `USD_per_Gbit_offered`, `Watt_per_Gbit_offered` | Cost or power divided by `alpha_star * base_total_demand`. | -| `USD_per_Gbit_p999`, `Watt_per_Gbit_p999` | Cost or power divided by absolute bandwidth at 99.9% availability. | - -Missing cost data is unavailable. Cost ratios are undefined when -their denominator is zero or unavailable. JSON writes undefined numeric metrics -as `null`, and CSV uses empty numeric fields. Reported zero cost or power stays zero. -Time per iteration is amortized over the requested failure samples plus baseline, -including repeated patterns; it is not the runtime of a single unique solve. -Timing depends on the execution environment; compare runs under similar conditions. - -## Project files - -| File | Contents | -|------|----------| -| `project.csv` | One row per scenario with the metrics above. | -| `project_per_seed_abs.csv` | Absolute per-seed metrics used by `abs_*.png` plots. | -| `project_baseline_normalized.csv` | Scenario summaries relative to the selected baseline. | -| `project_baseline_normalized_per_seed.csv` | Per-seed ratios (`*_r`, target 1) and deltas (`*_d`, target 0). | -| `normalized_insights.csv` | Means, common-seed counts, and paired-test p-values; Holm correction is applied across scenarios within each metric. | - -Normalized comparisons pair matching seeds with the baseline. The text summary -shows absolute metrics, normalized metrics, and comparison statistics. -`netlab test A B` runs paired t-tests between two scenarios on the per-seed -metrics in `_metrics`; `--alpha` sets the significance level. Inference -requires at least three finite matched seeds. Constant observed differences use -point CIs and p=0 (nonzero difference) or p=1 (zero); this computational convention -does not establish certainty about a population. Holm families contain available -scenario comparisons within each metric. Heatmap markers use adjusted p-values. - -The baseline is chosen consistently in tables and plots: an explicit argument, -then `NGRAPH_BASELINE_SCENARIO`, then a name containing `baseline`, then the first -alphabetically. An explicitly named missing baseline is an error. Ratios with -zero denominators remain undefined even in the baseline's own row. - -Pooling weights every sample equally: seeds with more iterations contribute more -weight. Seed medians and IQR use equal seed weight. Independent failure-pattern -positions are never aligned across seeds. - -## Figures - -| Figure | Contents | -|--------|----------| -| `BAC.png` | Pooled availability `P(delivered ≥ threshold)` versus percent of baseline delivery. IQR bands across seed curves are shown with at least three seeds. | -| `BAC_delta_vs_baseline.png` | Difference in availability from the selected baseline, by delivered-bandwidth threshold. Positive values mean higher availability. | -| `Latency_p99.png` | Pooled exceedance `P(iteration p99 stretch ≥ threshold)` across failure iterations. Lower values mean fewer iterations exceed the threshold. | -| `IterationOps.png` | Median and IQR across seeds for failure iterations, distinct patterns, and seconds per iteration. | -| `effects_heatmap.png` | Mean normalized effects; markers use Holm-adjusted p-values below 0.05. | -| `abs_*.png`, `norm_*.png` | Per-seed values and scenario medians, in absolute or baseline-normalized units. | - -## Scenario and seed files - -Each scenario directory contains `alpha_summary.json`, `bac_summary.json`, -`latency_summary.csv`, `iterops_summary.csv`, `costpower_summary.csv`, and -`network_stats_summary.csv`, plus scenario plots when enabled. - -Its `seed*/` directories contain metric JSON files, BAC sample CSVs, per-iteration -latency data, and placement matrices. [tests/data/README.md](tests/data/README.md) -describes the checked-in examples and how to regenerate them. - -## Reproducibility and interpretation - -Each `netlab metrics` invocation replaces the complete generated report tree, -including when `--only` selects a subset. Old seeds, optional metrics, and -plots are removed. A failing run preserves the previous report and its provenance. -Filename and recorded seeds must agree, source hashes must remain unchanged during -analysis, and provenance records numeric-library versions and analysis settings. - -Finite sample percentiles cannot establish multi-nine uptime. -MSD is bounded by its search settings and engine precision; -a disconnected network for which no positive alpha is feasible is rejected by -NetGraph. A supplied zero-baseline result can still be analyzed in absolute units, -with normalized metrics undefined. - -Reproduce the independent verification with: - -```sh -venv/bin/python dev/verify_metrics.py --plots -``` - -See [dev/METRICS_VERIFICATION.md](dev/METRICS_VERIFICATION.md) for evidence, -defects found, experiment scope, and validation results. +Stretch compares path cost with a reference cost; it does not measure elapsed +delay. Each source/destination pair uses the cheapest path that carried positive +volume in its baseline `cost_distribution`. If that cost is 2 and a failure +sends traffic along a path of cost 3, the stretch is `3/2 = 1.5`. + +Percentiles and traffic shares are weighted by delivered volume. A failure +summary takes the median across iterations. A project summary then takes the +median across seeds. A volume-weighted p99 is the smallest stretch that at least +99% of the included volume does not exceed. + +| Column | Meaning | +| --- | --- | +| `lat_base_p50` | Baseline median stretch | +| `lat_fail_p99` | Median of the failure iterations' volume-weighted p99 stretches | +| `lat_TD99` | Failure p99 summary divided by baseline p99 | +| `lat_SLO_1_2_drop` | Baseline share at stretch ≤1.2 minus the median failure share | +| `lat_best_path_drop` | Baseline share at the reference cost minus the median failure share | +| `lat_WES_delta` | Median failure weighted excess stretch minus baseline weighted excess stretch | + +In the column names, SLO means a service-level objective: here, stretch at most +1.2. Weighted excess stretch (WES) is the volume-weighted mean of +`max(stretch - 1, 0)`. +For equal delivered volumes at stretch 1 and 1.5, it is `(0 + 0.5)/2 = 0.25`. + +Dropped traffic has no delivered path cost. Latency metrics include only delivered +traffic with a positive baseline reference. The per-seed JSON field +`reference_coverage` says what fraction of delivered volume has that reference. +It is zero when path details are missing and undefined when nothing is delivered. +A total outage keeps a missing latency entry and is excluded from latency medians. + +Read latency with BAC and reference coverage. A low stretch after a failure can +coexist with large traffic loss. `lat_fail_p99` describes a typical iteration's +tail, not the worst failure or the p99 of all delivered traffic pooled together. + +## Iterations, timing and equipment + +| Column | Meaning | +| --- | --- | +| `iters_fail` | Sum of failure-pattern occurrence counts | +| `iters_total` | Failure iterations plus one baseline | +| `unique_patterns` | Number of distinct failure records | +| `tm_duration_total_sec` | Recorded `tm_placement` duration in seconds | +| `tm_duration_per_iter_sec` | That duration divided by total iterations | +| `capex_total` | Equipment purchase cost reported by `cost_power` | +| `USD_per_Gbit_offered`, `Watt_per_Gbit_offered` | Cost or power divided by `alpha_star * base_total_demand` | +| `USD_per_Gbit_p999`, `Watt_per_Gbit_p999` | Cost or power divided by absolute bandwidth at 99.9% availability | + +Missing equipment data is unavailable; reported zero cost or power stays zero. +Ratios are undefined if their denominator is zero or unavailable. Undefined +numbers are written as `null` in JSON and empty numeric fields in CSV. + +Time per iteration divides the whole recorded duration by the requested samples +plus baseline, including repeated failure patterns. It is an average accounting +value, not the measured runtime of one unique network solve. Compare timings only +under similar execution conditions. + +## Compare scenarios and seeds + +A project metric usually gives each seed equal weight. Pooled curves give each +sample equal weight, so a seed with more iterations contributes more observations. +For example, pooling a ten-sample seed and a hundred-sample seed gives the second +seed ten times as much weight. A median of their seed summaries treats them equally. +Failure sample positions are not matched across different seeds. + +Normalized tables compare matching seeds with a selected baseline scenario. +Ratios (`*_r`) have target 1; differences (`*_d`) have target 0. A zero or missing +denominator stays undefined, even in the baseline's own row. + +To choose the baseline scenario on the command line, set +`NGRAPH_BASELINE_SCENARIO=`; Python callers can pass the name directly. +Otherwise NetLab takes the first name, alphabetically, that contains `baseline` +in any letter case, or else the first name alphabetically. A requested baseline +that does not exist is an error. + +`netlab test A B` runs a paired t-test on the seeds that A and B share, +for six per-seed metrics: `alpha_star`, BW@99.9% as a fraction of baseline +delivery, `lat_fail_p99`, `USD_per_Gbit_p999`, `Watt_per_Gbit_p999` and +`capex_total`. `--alpha` sets the significance level. A metric is tested only +with at least three seeds where both values are finite, and the test assumes +those paired observations suit a t-test. Confidence intervals describe the +estimated mean difference. Constant observed differences use point intervals and +p=0 for a nonzero difference or p=1 for zero; this calculation does not +establish population-level certainty. `netlab test` compares one pair of +scenarios, so its adjusted p-value equals p. + +For each scenario other than the baseline, `normalized_insights.csv` tests +whether its per-seed ratios differ from 1 and its differences from 0. Holm's +multiple-comparison correction is applied across scenarios within each metric. +Heatmap markers use those adjusted p-values. + +## Output files + +The report is written next to the scenario root: `scenarios/` produces +`scenarios_metrics/`. + +| Project file | Contents | +| --- | --- | +| `project.csv` | One summary row per scenario | +| `project_per_seed_abs.csv` | Absolute per-seed metrics | +| `project_baseline_normalized.csv` | Scenario summaries relative to the selected baseline | +| `project_baseline_normalized_per_seed.csv` | Per-seed ratios and differences | +| `normalized_insights.csv` | Means, matched-seed counts and comparison statistics | + +| Figure | What to look for | +| --- | --- | +| `BAC.png` | Availability `P(delivered ≥ threshold)` versus percent of baseline delivery; higher is better | +| `BAC_delta_vs_baseline.png` | Availability difference from the selected baseline; positive means higher availability | +| `Latency_p99.png` | Fraction of failure iterations with p99 stretch at or above each threshold; lower is better | +| `IterationOps.png` | Median and interquartile range of iteration counts, distinct patterns and seconds per iteration | +| `effects_heatmap.png` | Mean normalized effects, with markers for adjusted p-values below 0.05 | +| `abs_*.png`, `norm_*.png` | Per-seed values and scenario medians in absolute or normalized units | + +The interquartile range is the interval from the 25th to the 75th percentile. +BAC seed bands are shown with at least three seeds. + +Each scenario has `alpha_summary.json`, `bac_summary.json`, +`latency_summary.csv`, `iterops_summary.csv`, `costpower_summary.csv` and +`network_stats_summary.csv`. Its `seed*/` directories contain metric JSON, +BAC samples, per-iteration latency data and placement matrices. +See [the saved test examples](tests/data/README.md) for regeneration instructions. + +## Save a repeatable report + +A metrics calculation replaces the complete report tree, even when `--only` +selects a subset. Removed seeds, optional metrics and plots are removed from the +report too. A failing calculation preserves the existing report and its provenance, +the record of inputs, settings and software versions used to produce it. + +The seed in a result filename (`__seed`) must match the `scenario.seed` +recorded inside the file. If a result file changes while the analysis runs, the +analysis fails. The provenance records numeric-library versions and settings. + +Finite failure samples cannot establish multi-nine uptime. MSD remains bounded +by its search settings and engine precision. NetGraph rejects an MSD search when +no positive multiplier is feasible. Supplied results with zero baseline delivery +can still be analyzed in absolute bandwidth units; their normalized values are +undefined. + +To check the calculations yourself, follow the +[metric verification guide](dev/METRICS_VERIFICATION.md). It uses small networks +with independently calculated expected answers. diff --git a/netlab/autoresearch/scenario_generator.py b/netlab/autoresearch/scenario_generator.py index fbfaef8..1533214 100644 --- a/netlab/autoresearch/scenario_generator.py +++ b/netlab/autoresearch/scenario_generator.py @@ -288,7 +288,6 @@ def _build_workflow(config: DcBbScenarioConfig) -> list[dict]: "type": "MaximumSupportedDemand", "name": "msd_baseline", "demand_set": "baseline_traffic_matrix", - "seed": config.seed, "resolution": config.msd_resolution, }, ] @@ -302,7 +301,6 @@ def _build_workflow(config: DcBbScenarioConfig) -> list[dict]: "failure_policy": f"fm_{name}", "iterations": config.failure_iterations, "parallelism": 8, - "seed": config.seed, "alpha_from_step": "msd_baseline", "alpha_from_field": "data.alpha_star", } @@ -316,7 +314,6 @@ def _build_workflow(config: DcBbScenarioConfig) -> list[dict]: "failure_policy": "fm_combined", "iterations": config.failure_iterations, "parallelism": 8, - "seed": config.seed, "alpha_from_step": "msd_baseline", "alpha_from_field": "data.alpha_star", } diff --git a/research_projects/dc-bb-autoresearch/program.md b/research_projects/dc-bb-autoresearch/program.md index 8d6c2f1..f29650b 100644 --- a/research_projects/dc-bb-autoresearch/program.md +++ b/research_projects/dc-bb-autoresearch/program.md @@ -1,120 +1,147 @@ -# DC-BB Interconnect Optimization - -## Topology - -Two data centers (ABC1 and XYZ1) connected through a shared 64-plane backbone. - -**ABC1 (DCType1):** 576 FADU devices in a 16×36 grid (16 HGRIDs × 36 FADU/HGRID). Up to 16 BB-facing ports per FADU. - -**XYZ1 (DCTypeF):** 1,536 XSW devices in a 64×24 grid (64 devices/plane × 24 planes). Up to 4 BB-facing ports per XSW. - -**Backbone:** 256 BB devices per site (64 planes × 4 devices/plane). Dual long-haul paths (Path_A, Path_B) cross-connect at 800 Gbps per link. Total cross-site: 2,048 links. - -## Mesh Group Interconnect - -G groups partition both DC and BB grids into equal rectangular blocks. Full mesh within each group. Notation `ArxBc <> CrxDc`: DC block (A rows × B cols) paired with BB block (C rows × D cols). - -BB block shape controls how connections overlap failure domains: planes, plane groups, and device indices. DC block shape controls which devices share a group. - -## Design Rules - -For single plane-site, device-index, and BB-device failures: - -1. Every DC device retains at least one BB connection. -2. Each device retains at least 75% of its BB connections. -3. Maximize G subject to these rules to reduce port count. - -The structural feasibility check excludes plane-group and long-haul failures. -Evaluate those separately in simulations; passing the structural check does not -establish resilience to every failure mode. - -## Reference results - -These tables are study inputs without saved run provenance. Reproduce the selected -configurations before using the values to rank designs. The current generator -runs each configured failure mode separately and combines them with equal weight. -Record the NetGraph version, seed, iteration count, and policies for new results. - -Metrics: - -- **alpha_star**: Demand multiplier found within the configured MSD search bounds. -- **BAC AUC**: Mean delivered/baseline bandwidth, capped at 1, over baseline and failure samples. - -### ABC1 Results (other side fixed at G_xyz1=64) - -| G | k_dc | BB Block | alpha* | BAC AUC | Feasible | -|---|------|----------|--------|---------|----------| -| 16 | 16 | 16rx1c | 9.21 | 1.0000 | ✓ | -| 16 | 16 | 4rx4c | 9.21 | 1.0000 | ✓ | -| 16 | 16 | 8rx2c | 9.21 | 0.9994 | ✓ | -| 32 | 8 | 8rx1c | 9.21 | 1.0000 | ✗ | -| 32 | 8 | 4rx2c | 9.21 | 1.0000 | ✗ | -| 32 | 8 | 2rx4c | 9.21 | 1.0000 | ✗ | -| 64 | 4 | 4rx1c | 9.21 | 0.8900 | ✗ | -| 64 | 4 | 2rx2c | 9.21 | 0.8885 | ✗ | -| 64 | 4 | 1rx4c | 9.21 | 0.8891 | ✗ | - -The listed ABC1 cases have alpha 9.21. Their BAC differs by group count: about -1.0 for G=16/32 and 0.89 for G=64. Check the simulated link utilization before -attributing the shared alpha to a bottleneck. - -### XYZ1 Results (other side fixed at G_abc1=64) - -| G | k_dc | BB Block | alpha* | BAC AUC | Feasible | -|---|------|----------|--------|---------|----------| -| 64 | 4 | 4rx1c | 9.21 | 0.8900 | ✗ | -| 64 | 4 | 2rx2c | 9.21 | 0.8851 | ✗ | -| 64 | 4 | 1rx4c | 9.21 | 0.8943 | ✗ | -| 128 | 2 | 2rx1c | 9.21 | 0.8697 | ✗ | -| 128 | 2 | 1rx2c | 9.21 | 0.8697 | ✗ | -| 256 | 1 | 1rx1c | 6.14 | 0.9405 | ✗ | - -In these inputs, XYZ1 G=256 has lower alpha and higher normalized BAC than -G=64/128. Its DC-BB capacity is 1,536 × 400 Gbps = 614.4 Tbps. Compare absolute -delivered bandwidth as well as normalized BAC. None of the listed XYZ1 layouts -passes the connection-retention rules. - -### Cross-Side Results (54 combinations, top by BAC) - -| G_abc1 | BB_abc1 | G_xyz1 | BB_xyz1 | alpha* | BAC AUC | -|--------|---------|--------|---------|--------|---------| -| 16/32 | any | 64 | any | 9.21 | 1.0000 | -| 16/32 | any | 128 | any | 9.21 | 0.9651-0.9660 | -| 64 | 1rx4c | 256 | 1rx1c | 6.14 | 0.9511 | -| 16 | 16rx1c | 256 | 1rx1c | 6.14 | 0.9488 | -| 64 | any | 64 | any | 9.21 | 0.8849-0.8946 | -| 64 | any | 128 | any | 9.21 | 0.8694-0.8700 | - -## Research Questions - -Use fresh simulation results to answer: - -1. **Deployment recommendation:** Which ABC1 × XYZ1 combination warrants further testing? Consider the tradeoff between port cost (higher G = fewer ports), capacity (alpha), and resilience (BAC). - -2. **G=32 on ABC1:** It achieves BAC = 1.0 like G=16 but uses half the BB ports (8 vs 16 per FADU). The structural analysis flags it as infeasible under the connection-retention rules. Is the simulation BAC of 1.0 trustworthy, or is 200 iterations insufficient to reveal the vulnerability? - -3. **XYZ1 G=256 vs G=64:** Is the 33% capacity reduction acceptable for the BAC improvement? Under what traffic load assumptions? - -4. **DC-side block choice:** For the recommended G values, does the DC-side factorization matter for operational reasons (cabling, traffic locality, failure blast radius)? +# Data-center-to-backbone interconnect design + +Choose how data-center (DC) devices connect to backbone (BB) devices at two +sites, called ABC1 and XYZ1. Each proposal picks a group count and a layout for +each site. NetLab builds that network, runs it in NetGraph and reports the result. +Compare traffic capacity, delivery under failures and the number of ports needed. +A design is not better just because it has fewer links or a higher normalized score. + +## The network being modeled + +| Part | Grid and devices | Backbone ports per DC device | +| --- | --- | --- | +| ABC1 | 16 rows × 36 devices = 576 DC devices, named FADU in the model | At most 16 | +| XYZ1 | 64 device positions × 24 planes = 1,536 DC devices, named XSW in the model | At most 4 | +| Backbone at each site | 64 planes × 4 devices = 256 devices | | + +A plane is a set of backbone devices that can fail together. DC-to-backbone links +carry 400 Gbps. The two backbones are joined plane by plane: every backbone +device in a plane at ABC1 connects to every device in the same plane at XYZ1, once +on each of two long-haul paths. That gives `64 × 4 × 4 × 2 = 2,048` long-haul +links of 800 Gbps. + +The traffic is 100,000 Gbps from the ABC1 rack switches to XYZ1 and 100,000 Gbps +back. `alpha_star` multiplies both directions: `alpha_star = 2` means 200,000 Gbps +each way. The traffic also crosses each site's internal switches, which have their +own capacities. These values come from `DcBbScenarioConfig` in +`netlab/autoresearch/dcbb_config.py` and from +`netlab/autoresearch/scenario_generator.py`. + +## Divide devices into connection groups + +Each side divides both its DC grid and its BB grid into `G` rectangular groups. +Every device belongs to one group. Within a group, every DC device connects to +every BB device. A DC device therefore has `256 / G` backbone links, and a side +has `DC devices × 256 / G` DC-to-BB links. + +For example, ABC1 with `G=16` has `576/16 = 36` DC devices and `256/16 = 16` BB +devices per group. Each DC device needs 16 BB-facing ports, for +`576 × 16 = 9,216` DC-to-BB links on that side. With `G=32`, each DC device needs +8 such ports. The port limits are why ABC1 needs `G` of at least 16 and XYZ1 at +least 64. + +A layout string has the form +`DC_row_groups x DC_column_groups _ BB_row_groups x BB_column_groups`. +The two products must both equal `G`, and each count must divide its grid +dimension. These are partition counts, not block sizes. The BB grid has 64 rows, +one per plane, and 4 columns, one per device position. + +For ABC1, `4x4_16x1` at `G=16` divides the DC grid into 4×4 groups of 4×9 +devices. It divides the BB grid into 16×1 groups of 4×4 devices: 4 planes and all +4 device positions. Matching groups are fully connected. For XYZ1, `16x4_16x4` at +`G=64` gives DC blocks of 4×6 and BB blocks of 4×1. + +## Check failure resilience + +The structural check calculates connection loss from the layout without running +traffic. It looks at three failures: one plane at one site, one device position +across a plane group of four consecutive planes, and one BB device. A layout +passes when, under each of them, both of these hold: + +1. Every DC device keeps at least one BB connection. +2. Every DC device keeps at least 75% of its BB connections. + +For ABC1 `4x4_16x1`, a DC device has 16 links to a block of 4 planes × 4 devices. +Losing one plane removes 4 of them (25%). Losing one device position across the +plane group also removes 4, and losing one BB device removes 1. The device keeps +75%, so the layout passes. For ABC1 `8x4_16x2` at `G=32`, the block is 4 planes × +2 devices; losing one device position removes 4 of 8 links, so it fails. + +Under these rules, every ABC1 layout with `G=16` passes, and no ABC1 layout with +`G=32` or `G=64` does. No XYZ1 layout passes. Keeping 75% of at most 4 links +allows each failure to remove only one link of four, so only `G=64` could pass. A +4×1 block loses one link to a plane failure but all four to a device-position +failure; 2×2 and 1×4 blocks lose two or four links to one plane. With `G=128` or +`G=256`, one BB device already removes half or all of an XSW's links. Compare +XYZ1 layouts by their simulated delivery. + +Prefer the largest `G` that passes, because a larger `G` uses fewer ports. The +structural rules leave out failures of a whole plane group and of a long-haul +path. Simulate those. A structural pass does not prove that traffic is delivered +under every failure mode. + +## How a proposal is evaluated + +NetLab generates the scenario for your parameters, checks it and runs NetGraph. +The workflow first runs `msd_baseline`, a `MaximumSupportedDemand` search for +`alpha_star`. It then places the traffic at that multiplier under sampled +failures: one placement step for each failure mode defined in +`netlab/autoresearch/dcbb_failures.py`, plus `tm_combined`, whose samples pick one +of those modes with equal probability. The modes cover long-haul paths, planes, +plane groups, device positions, one to eight BB devices and random outages of BB +devices and links. Every step derives its random seed from the run's seed +(`netlab autoresearch run --seed`), so all proposals in a run use the same seed. + +The project's [objective.yml](objective.yml) maximizes `alpha_star` and has no +constraints. Your score is `alpha_star` alone: the connection-retention rules and +failure delivery are not part of it, so weigh them yourself. Results that fail to +generate or validate are reported with their error. + +`alpha_star` is the demand multiplier found by the bounded MSD search. BAC AUC is +the mean delivered/baseline ratio, capped at 1, over the baseline and the failure +samples. Also compare absolute delivered bandwidth: a smaller network can have a +higher normalized BAC while carrying less traffic. See +[the metric definitions](../../metrics.md). + +## Run comparisons from the command line + +From the NetLab repository root with the development environment active: + +```bash +netlab autoresearch structural-analysis --output layouts.json +netlab autoresearch sweep abc1 --output-dir results/abc1 +netlab autoresearch cross-sweep --output-dir results/cross +``` -5. **Sensitivity to failure weights:** How does the ranking change when the combined policy weights change? Report the weights used and compare per-mode results. +`structural-analysis` prints every layout's connection loss for each failure type +and saves it to `layouts.json`. `sweep abc1` simulates one layout for each distinct +combination of `G` and BB block shape on ABC1, with XYZ1 held at `G=64` and +`16x4_16x4`. `sweep xyz1` does the same for XYZ1, with ABC1 held at `G=64` and +`16x4_16x4`. `cross-sweep` simulates every combination of those ABC1 and XYZ1 +choices. Each entry in `results.jsonl` records `alpha_star`, the BAC AUC of +`tm_combined` and the BAC of every failure mode. The output directory's +`sweep.json` records the seed, iteration count and package versions. -## Parameters +## Questions to answer -- `g_abc1`: 16, 32, 64 -- `g_xyz1`: 64, 128, 256 -- `layout_abc1`: Grid partition counts; both products must equal `g_abc1`. -- `layout_xyz1`: Grid partition counts; both products must equal `g_xyz1`. +- Which layouts satisfy the connection-retention rules, and how many ports do they need? +- Do those layouts still deliver the required traffic under plane-group and long-haul failures? +- Does increasing `G` save ports at an acceptable cost in absolute capacity and delivered bandwidth? +- How does the block shape change capacity and delivery for the same `G`? +- How would different failure-mode weights change the ranking? -A layout is `DC_row_groups x DC_column_groups _ BB_row_groups x BB_column_groups`. -Block dimensions are grid dimensions divided by these group counts. +## Propose parameters -Return parameter choices as YAML: +Choose values from the parameter space listed with each request; it comes from +[hypothesis_template.yml](hypothesis_template.yml). Group counts are integers: +16, 32 or 64 for ABC1 and 64, 128 or 256 for XYZ1. A layout must match its chosen +group count. Return the choice as YAML, with the group counts written as plain +numbers, not quoted strings: ```yaml params: - g_abc1: "16" - g_xyz1: "64" + g_abc1: 16 + g_xyz1: 64 layout_abc1: "4x4_16x1" layout_xyz1: "16x4_16x4" ``` diff --git a/tests/autoresearch/test_generate_scenario.py b/tests/autoresearch/test_generate_scenario.py index 1e4f12c..45afe13 100644 --- a/tests/autoresearch/test_generate_scenario.py +++ b/tests/autoresearch/test_generate_scenario.py @@ -7,11 +7,43 @@ from netlab.autoresearch.dcbb_config import DcBbScenarioConfig, validate_config from netlab.autoresearch.scenario_generator import ( + generate_from_parameters, generate_scenario, generate_scenario_with_validation, ) +@pytest.mark.parametrize("programmatic", [False, True]) +def test_workflow_randomness_follows_run_seed(programmatic): + """Both generator entry points honor the scenario's final run seed.""" + from ngraph.utils.seed_manager import SeedManager + from ngraph.workflow.parse import build_workflow_steps + + def resolve(run_seed): + if programmatic: + scenario = generate_from_parameters() + # The runner supplies --seed after generating the hypothesis. + scenario["seed"] = run_seed + else: + scenario = generate_scenario(DcBbScenarioConfig(seed=run_seed)) + seeds = SeedManager(scenario["seed"]) + steps = build_workflow_steps( + scenario["workflow"], + derive_seed=lambda name: seeds.derive_seed("workflow_step", name), + ) + actual = {step.name: step.seed for step in steps} + expected = { + step.name: seeds.derive_seed("workflow_step", step.name) for step in steps + } + assert actual == expected + return actual + + first = resolve(7) + assert resolve(7) == first + second = resolve(42) + assert all(first[name] != second[name] for name in first) + + class TestValidateConfig: """validate_config returns errors for invalid configs, empty for valid.""" diff --git a/tests/data/README.md b/tests/data/README.md index 38bc077..4c935e5 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -1,38 +1,51 @@ -# Simulation fixtures +# Saved networks and metric examples -| Fixture | Purpose | +The tests use these small scenarios and saved results to check simulation, +metric calculation, tables and figures without downloading geographic datasets. +A fixture is one of these saved inputs or expected outputs. + +| Fixture | Used for | | --- | --- | -| `mini_dcbb.yaml` | Live verification against hand-calculated expectations | -| `mini_dcbb_output/mini_dcbb.results.json` | Analysis-loop tests | -| `../autoresearch/data/square_mesh_results.json` | Autoresearch and metrics tests | -| `scenarios/` | Four topology configurations, each built for seeds 11 and 12 | -| `scenarios_metrics/` | CLI, integration, summary, and plotting references | +| `mini_dcbb.yaml` | A small data-center/backbone network with hand-calculated answers | +| `mini_dcbb_output/mini_dcbb.results.json` | Research analysis-loop tests | +| `../autoresearch/data/square_mesh_results.json` | Research and metric calculations on a mesh network | +| `scenarios/` | Baseline, Clos and Dragonfly configurations built with seeds 11 and 12 | +| `scenarios_metrics/` | Expected CLI summaries, tables and figures | + +## Regenerate the files -Regenerate from an environment installed with the dependencies declared in -`pyproject.toml`: +From the repository root, install the dependencies declared in `pyproject.toml` +and run: ```bash venv/bin/python dev/regenerate_fixtures.py ``` -The script builds `topogen_configs_small` from the four cached corridor MultiGraphs, -executes NetGraph, and computes the metrics. It also runs the mini DC-BB and -square-mesh scenarios. All API tasks finish successfully before the fixtures are -replaced. Logs, the previous fixtures, and the recorded provenance are saved -under `build/fixture-regeneration/`. Installed sources must be released versions -or clean Git revisions; the script refuses editable checkouts with uncommitted -changes. - -Each small scenario uses 1,000 failure draws plus one baseline. Its workflow -inherits the scenario seed. Demand order is seed-dependent and affects greedy -placement within a priority: alpha is 1.40625 for seed 11 and 1.6875 for seed 12. -The metrics weight each failure pattern by its `occurrence_count`. Timing values -record execution duration and are not performance assertions. - -Current fixtures were generated with Python 3.13, `ngraph==0.24.0` and -`netgraph-core==0.11.0` from PyPI, and TopoGen commit -`3bf0fa4aa08604a886421461f7ce36dbd7275121`. - -The cached geography uses corridor MultiGraphs. BAC curves store unique thresholds -and inclusive `P(delivered >= threshold)` values; sample quantiles use all -occurrence-weighted draws. +The script uses the saved corridor graphs to build `topogen_configs_small`, +runs NetGraph and computes metrics. It also runs the mini data-center/backbone +and square-mesh scenarios. All simulation tasks must succeed before it replaces +the fixtures. + +The script saves logs, a copy of the existing fixtures and input/software records +under `build/fixture-regeneration/`. Editable source checkouts must have no +uncommitted changes; released installations are also accepted. Run regeneration +in a clean checkout rather than discarding work to satisfy that check. + +## What the examples exercise + +Each small scenario places its traffic under 1,000 sampled failures plus a +no-failure baseline. Its workflow inherits the scenario seed. The seed can change +demand order; when placement processes demands in order, that can also change +the accepted traffic multiplier. In the saved results, all four configurations +have `alpha_star` 1.40625 with seed 11 and 1.6875 with seed 12, for the same +90,000 Gbps base traffic. + +Metrics weight each pattern by `occurrence_count`. BAC curves store distinct +bandwidth thresholds and the fraction of samples at or above each threshold. +Repeated observations remain part of the calculation. Saved durations describe +one execution environment and are not performance expectations. + +The accompanying `provenance.json` files record package versions, input hashes +and analysis settings. Read them when comparing regenerated outputs. A fixture's +recorded dependency version documents its origin; it does not set the package's +supported versions, which are declared in [pyproject.toml](../../pyproject.toml). diff --git a/tests/test_dcbb_experiment.py b/tests/test_dcbb_experiment.py new file mode 100644 index 0000000..ed4506e --- /dev/null +++ b/tests/test_dcbb_experiment.py @@ -0,0 +1,122 @@ +"""Keep the experiment workflow, seed summary and comparison consistent.""" + +from __future__ import annotations + +import importlib.util +import json +import re +from pathlib import Path + +import pytest +import yaml +from ngraph.scenario import Scenario + +from netlab.simulation import simulate + +EXPERIMENT = Path(__file__).resolve().parents[1] / "experiments" / "dc-bb-interconnect" +TOPOLOGY = "dc4x9_bb4x4_bb4x4_dc4x9_one_to_one" + + +@pytest.fixture +def experiment_module(monkeypatch): + monkeypatch.syspath_prepend(str(EXPERIMENT)) + spec = importlib.util.spec_from_file_location( + "dcbb_experiment_runner", EXPERIMENT / "run.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def save_result(runner, scenario, seed, results): + directory = runner.results_dir / scenario / f"{scenario}__seed{seed}" + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{scenario}__seed{seed}_scenario.results.json" + path.write_text(json.dumps(results)) + return path + + +def read_summary(runner, scenario): + return json.loads((runner.results_dir / scenario / "summary.json").read_text()) + + +def test_included_workflow_preserves_msd_in_summary_and_comparison( + experiment_module, tmp_path, capsys +): + runner = experiment_module.DcBbExperimentRunner(EXPERIMENT, results_dir=tmp_path) + document = runner.merger.merge( + EXPERIMENT / "topologies" / TOPOLOGY / "scenario.yml", seed=42 + ) + for step in document["workflow"]: + if "iterations" in step: + step.update(iterations=1, parallelism=1) + outcome = simulate(yaml.safe_dump(document, sort_keys=False)) + assert outcome.success, outcome.error + msd_name = next( + step["name"] + for step in document["workflow"] + if step["type"] == "MaximumSupportedDemand" + ) + alpha = outcome.results["steps"][msd_name]["data"]["alpha_star"] + assert alpha > 0 + save_result(runner, TOPOLOGY, 42, outcome.results) + runner.compute_metrics([TOPOLOGY]) + assert read_summary(runner, TOPOLOGY)["alpha_star"] == alpha + runner.generate_comparison([TOPOLOGY]) + comparison = json.loads((tmp_path / "comparison.json").read_text()) + assert comparison["data"][TOPOLOGY]["alpha_star"] == alpha + assert f"{alpha:.2f}" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "alphas, expected", [([1, 3, 9], 3), ([0, 0, 9], 0), ([0], 0), ([None], None)] +) +def test_summary_aggregates_seed_alphas_and_keeps_zero( + experiment_module, tmp_path, alphas, expected +): + runner = experiment_module.DcBbExperimentRunner(EXPERIMENT, results_dir=tmp_path) + for seed, alpha in enumerate(alphas, 1): + result = {"steps": {"msd_baseline": {"data": {"alpha_star": alpha}}}} + save_result(runner, "example", seed, result) + runner.compute_metrics(["example"]) + summary = read_summary(runner, "example") + assert summary["alpha_star"] == expected + assert summary["alpha_star_by_seed"] == { + str(seed): alpha for seed, alpha in enumerate(alphas, 1) + } + + +def test_two_to_one_name_matches_expanded_backbone(experiment_module, tmp_path): + runner = experiment_module.DcBbExperimentRunner(EXPERIMENT, results_dir=tmp_path) + name = next(n for n in runner.discover_scenarios() if n.endswith("_two_to_one")) + dimensions = [ + tuple(map(int, shape)) for shape in re.findall(r"bb(\d+)x(\d+)", name) + ] + document = runner.merger.merge(EXPERIMENT / "topologies" / name / "scenario.yml") + network = Scenario.from_yaml(yaml.safe_dump(document)).network + for site, (planes, devices) in zip(["SiteA", "SiteB"], dimensions, strict=True): + nodes = [n for n in network.nodes if n.startswith(f"{site}/BB-")] + assert len(nodes) == planes * devices + assert len({n.split("/")[1] for n in nodes}) == planes + + +@pytest.mark.parametrize("other_alpha", [None, 2]) +def test_comparison_displays_zero_and_handles_zero_maximum( + experiment_module, tmp_path, capsys, other_alpha +): + runner = experiment_module.DcBbExperimentRunner(EXPERIMENT, results_dir=tmp_path) + for scenario, alpha in [("zero", 0), ("other", other_alpha)]: + directory = tmp_path / scenario + directory.mkdir() + (directory / "summary.json").write_text( + json.dumps({"alpha_star": alpha, "network": {}}) + ) + runner.generate_comparison(["zero", "other"]) + row = next( + line + for line in capsys.readouterr().out.splitlines() + if line.startswith("│ zero ") + ).split() + assert row[3] == "0.00" + assert row[4] == ("0.0%" if other_alpha else "?")