From 6f935c4c673e19e5115b69c509989de363507eeb Mon Sep 17 00:00:00 2001 From: iharhl Date: Fri, 13 Nov 2026 22:26:20 +0100 Subject: [PATCH] Add from-partitions to draw ESP-IDF partition tables `memmap from-partitions TABLE` writes a config from an ESP-IDF partition table, the CSV or the partition-table.bin the build writes, then checks it. It places partitions as ESP-IDF's gen_esp32part.py does, and draws one flash column with the bootloader, the partition table, each partition and the free space. --sdkconfig gives the flash size and the partition table and bootloader offsets; --flash-size, --table-offset and --bootloader-offset stand in for it. A CSV that breaks a rule of ESP-IDF's table check gets the new W113. Tests run on five real ESP-IDF builds, whose partition-table.bin is the ground truth for placing their CSV, and five tables from ESP-IDF, Arduino-ESP32 and MicroPython. docs/from-partitions.md describes the command and its limits. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 33 +- README.md | 41 +- docs/from-partitions.md | 166 ++++ memmap/cli.py | 121 ++- memmap/from_elf/writer.py | 34 +- memmap/from_partitions/__init__.py | 11 + memmap/from_partitions/esp_idf.py | 322 ++++++++ memmap/from_partitions/importer.py | 385 +++++++++ memmap/from_partitions/sdkconfig.py | 84 ++ memmap/from_partitions/writer.py | 60 ++ .../partition-table.bin | Bin 0 -> 3072 bytes .../esp32_homekit_custom_type/partitions.csv | 10 + .../esp32_homekit_custom_type/sdkconfig | 9 + .../esp32_small_nvs/partition-table.bin | Bin 0 -> 3072 bytes .../builds/esp32_small_nvs/partitions.csv | 12 + .../builds/esp32_small_nvs/sdkconfig | 12 + .../partition-table.bin | Bin 0 -> 3072 bytes .../builds/esp32_table_at_0x18000/sdkconfig | 9 + .../builds/esp32h2_zigbee/partition-table.bin | Bin 0 -> 3072 bytes .../builds/esp32h2_zigbee/partitions.csv | 7 + .../builds/esp32h2_zigbee/sdkconfig | 13 + .../esp32p4_singleapp/partition-table.bin | Bin 0 -> 3072 bytes .../builds/esp32p4_singleapp/sdkconfig | 14 + tests/fixtures/esp_partitions/index.json | 123 +++ tests/fixtures/esp_partitions/refresh.py | 169 ++++ .../esp_partitions/tables/arduino_default.csv | 7 + .../tables/idf_ota_efuse_emul.csv | 11 + .../tables/idf_partitions_singleapp.csv | 5 + .../tables/idf_partitions_two_ota.csv | 8 + .../tables/micropython_4MiBplus.csv | 10 + tests/golden/partitions/arduino_default.yaml | 38 + .../partitions/esp32_homekit_custom_type.svg | 71 ++ .../partitions/esp32_homekit_custom_type.yaml | 43 + tests/golden/partitions/esp32_small_nvs.yaml | 42 + .../partitions/esp32_table_at_0x18000.yaml | 36 + tests/golden/partitions/esp32h2_zigbee.svg | 49 ++ tests/golden/partitions/esp32h2_zigbee.yaml | 38 + .../golden/partitions/esp32p4_singleapp.yaml | 36 + .../golden/partitions/idf_ota_efuse_emul.yaml | 42 + .../partitions/idf_partitions_singleapp.yaml | 36 + .../partitions/idf_partitions_two_ota.yaml | 39 + .../partitions/micropython_4MiBplus.yaml | 36 + tests/test_from_partitions.py | 739 ++++++++++++++++++ 43 files changed, 2895 insertions(+), 26 deletions(-) create mode 100644 docs/from-partitions.md create mode 100644 memmap/from_partitions/__init__.py create mode 100644 memmap/from_partitions/esp_idf.py create mode 100644 memmap/from_partitions/importer.py create mode 100644 memmap/from_partitions/sdkconfig.py create mode 100644 memmap/from_partitions/writer.py create mode 100644 tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partition-table.bin create mode 100644 tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partitions.csv create mode 100644 tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/sdkconfig create mode 100644 tests/fixtures/esp_partitions/builds/esp32_small_nvs/partition-table.bin create mode 100644 tests/fixtures/esp_partitions/builds/esp32_small_nvs/partitions.csv create mode 100644 tests/fixtures/esp_partitions/builds/esp32_small_nvs/sdkconfig create mode 100644 tests/fixtures/esp_partitions/builds/esp32_table_at_0x18000/partition-table.bin create mode 100644 tests/fixtures/esp_partitions/builds/esp32_table_at_0x18000/sdkconfig create mode 100644 tests/fixtures/esp_partitions/builds/esp32h2_zigbee/partition-table.bin create mode 100644 tests/fixtures/esp_partitions/builds/esp32h2_zigbee/partitions.csv create mode 100644 tests/fixtures/esp_partitions/builds/esp32h2_zigbee/sdkconfig create mode 100644 tests/fixtures/esp_partitions/builds/esp32p4_singleapp/partition-table.bin create mode 100644 tests/fixtures/esp_partitions/builds/esp32p4_singleapp/sdkconfig create mode 100644 tests/fixtures/esp_partitions/index.json create mode 100644 tests/fixtures/esp_partitions/refresh.py create mode 100644 tests/fixtures/esp_partitions/tables/arduino_default.csv create mode 100644 tests/fixtures/esp_partitions/tables/idf_ota_efuse_emul.csv create mode 100644 tests/fixtures/esp_partitions/tables/idf_partitions_singleapp.csv create mode 100644 tests/fixtures/esp_partitions/tables/idf_partitions_two_ota.csv create mode 100644 tests/fixtures/esp_partitions/tables/micropython_4MiBplus.csv create mode 100644 tests/golden/partitions/arduino_default.yaml create mode 100644 tests/golden/partitions/esp32_homekit_custom_type.svg create mode 100644 tests/golden/partitions/esp32_homekit_custom_type.yaml create mode 100644 tests/golden/partitions/esp32_small_nvs.yaml create mode 100644 tests/golden/partitions/esp32_table_at_0x18000.yaml create mode 100644 tests/golden/partitions/esp32h2_zigbee.svg create mode 100644 tests/golden/partitions/esp32h2_zigbee.yaml create mode 100644 tests/golden/partitions/esp32p4_singleapp.yaml create mode 100644 tests/golden/partitions/idf_ota_efuse_emul.yaml create mode 100644 tests/golden/partitions/idf_partitions_singleapp.yaml create mode 100644 tests/golden/partitions/idf_partitions_two_ota.yaml create mode 100644 tests/golden/partitions/micropython_4MiBplus.yaml create mode 100644 tests/test_from_partitions.py diff --git a/AGENTS.md b/AGENTS.md index d7e3943..d63716e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,24 @@ which keep the sections as `children` and each get a detail column. Zooms go one when W110 puts the regions in zooms of their own, their groups get a note instead. `--min-size 0` turns folding off. +## Typical task: map an ESP-IDF partition table +1. Find the table: the CSV that `CONFIG_PARTITION_TABLE_FILENAME` in the project's `sdkconfig` + names, or the `build/partition_table/partition-table.bin` the build writes from it. +2. `memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig --json --strict`. + The table does not record the flash size, so give `--sdkconfig`, which also gives the + partition table and bootloader offsets. Without one, give `--flash-size 4M`, + `--bootloader-offset` (0x1000 on the ESP32), and `--table-offset` when + `CONFIG_PARTITION_TABLE_OFFSET` is not 0x8000. +3. Fix W113 (a rule current ESP-IDF rejects, checked for a CSV only) by its hint. W100 marks a + partition edge inside a 4K erase sector. +4. Continue from step 4 of the first task. + +Partitions are placed as ESP-IDF places them: a blank offset follows the partition above it, +aligned to 64K for an app and 4K for the rest, starting after the partition table. Offsets are +positions in the flash chip, drawn from 0, not CPU addresses. The column also holds the +bootloader, the partition table and `Free` blocks; the header comments say where each setting +came from. https://github.com/iharhl/memmap/blob/main/docs/from-partitions.md has the details. + ## Rules - Never guess keys; run `memmap schema` when unsure. - Ranges are inclusive: end = start + size - 1. @@ -83,11 +101,12 @@ when W110 puts the regions in zooms of their own, their groups get a note instea Quote `--set` values that contain `[` or `{`, since shells expand them. ## JSON output -With `--json`, `check`, `render` and `from-elf` always print one object on stdout, for -validation errors and for usage and IO failures alike: `{"ok": ..., "diagnostics": [...]}`. -`render` adds `output` and `format` on success, `from-elf` adds `output` once it has written the -config. `describe --json` prints the same object on failure. Only a mistake in the command line -itself, such as an unknown flag, is reported as plain text on stderr. +With `--json`, `check`, `render`, `from-elf` and `from-partitions` always print one object on +stdout, for validation errors and for usage and IO failures alike: +`{"ok": ..., "diagnostics": [...]}`. `render` adds `output` and `format` on success, and the two +importers add `output` once they have written the config. `describe --json` prints the same +object on failure. Only a mistake in the command line itself, such as an unknown flag, is +reported as plain text on stderr. ## Checking the result `memmap describe layout.yaml --json` returns: @@ -147,6 +166,7 @@ Every diagnostic has a stable code, a YAML path, a line and usually a `hint`. In | W110 | `from-elf`: section outside every memory region, placed at the top level | | W111 | `from-elf`: section overlaps another one, left out | | W112 | `from-elf`: map region left out (no size, past the address space, overlap, too small) | +| W113 | `from-partitions`: the CSV breaks a rule of ESP-IDF's partition table check | | E090 | Usage or IO failure, such as a missing file or bad `--set` (`--json` only) | | E091 | Render or internal failure, such as missing cairo (`--json` only) | @@ -161,5 +181,8 @@ Exit codes: `0` ok, `1` validation errors, `2` usage or IO error, `3` render err real-firmware goldens: `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py tests/test_linker.py`. docs/from-elf.md describes them and how to add an image or a map. +- After an intended change to `from-partitions` or the drawing, regenerate its goldens: + `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_partitions.py`. docs/from-partitions.md + describes the real ESP-IDF builds they come from. - A new diagnostic code or recipe goes in this file too. `memmap guide` ships it to users, and `tests/test_cli.py` covers the recipes. diff --git a/README.md b/README.md index 05d1933..94b8dd8 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # memmap Draw datasheet-style memory maps of MCU memory. You describe the layout in a YAML file, or -have `memmap` write that file from a linked firmware image. Either way you get the same config, -which `memmap` checks and renders to SVG, PNG or PDF. +have `memmap` write that file from a linked firmware image or an ESP-IDF partition table. Either +way you get the same config, which `memmap` checks and renders to SVG, PNG or PDF. - **Write it by hand.** One YAML file holds the memory (`memory`) and how to draw it (`view`), kept separate. Start from one of the [examples](#examples). - **Generate it from firmware.** `from-elf` reads a linked ELF and the linker's map file (GNU ld, Keil armlink or IAR), and places every section at its real address and size in the chip's flash and RAM regions. +- **Generate it from a partition table.** `from-partitions` reads an ESP-IDF partition table, CSV + or binary, and places every partition where the ESP-IDF build would. - **Check before drawing.** Validation catches overlaps, overflow and flash sector misalignment, with the YAML path, line and a hint for each. - **Verify as text.** Output is deterministic, and `describe` lists everything drawn, so agents and @@ -45,7 +47,7 @@ export DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib ## Quick start -Choose a starting point. Both give you a `layout.yaml`. +Choose a starting point. Each gives you a `layout.yaml`. **A layout you are planning.** Copy the closest example: @@ -59,6 +61,12 @@ cp examples/custom_bootloader.yaml layout.yaml memmap from-elf build/firmware.elf -o layout.yaml --map build/firmware.map ``` +**An ESP-IDF project.** Pass its partition table and its `sdkconfig`: + +```bash +memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig +``` + Then check the config, see what will be drawn, and render it: ```bash @@ -130,6 +138,24 @@ ordinary config: rename or merge blocks, add `view` options, then `render` it. [docs/from-elf.md](docs/from-elf.md) covers what is read from each map, what happens when it is not enough, and how sections become blocks. +## Generating a config from an ESP-IDF partition table + +`from-partitions` draws the flash of an ESP-IDF project: the bootloader, the partition table, +each partition and the free space between them. It reads the partition table CSV the project +names in `CONFIG_PARTITION_TABLE_FILENAME`, or the `build/partition_table/partition-table.bin` +the build writes. Partitions with a blank offset are placed as the build places them, so the +drawing matches the binary table. + +The table does not record the flash size, so pass the project's `sdkconfig`, which also gives +the partition table and bootloader offsets. Without one, give the flash size: + +```bash +memmap from-partitions partitions.csv -o layout.yaml --flash-size 4M +``` + +A CSV that breaks a rule of ESP-IDF's own table check gets W113, with a hint. +[docs/from-partitions.md](docs/from-partitions.md) covers what is read and drawn, and the limits. + ## Commands | Command | Purpose | @@ -138,6 +164,7 @@ not enough, and how sections become blocks. | `memmap check CONFIG [--json] [--strict]` | Validate only | | `memmap describe CONFIG [--json]` | Resolved regions, blocks, labels and option origins | | `memmap from-elf ELF [-o OUT] [--map FILE] [--region id=start:size[:name]] [--min-size N] [--force] [--json] [--strict]` | Write a config from a linked ELF, then check it | +| `memmap from-partitions TABLE [-o OUT] [--sdkconfig FILE] [--flash-size SIZE] [--table-offset OFFSET] [--bootloader-offset OFFSET] [--force] [--json] [--strict]` | Write a config from an ESP-IDF partition table, then check it | | `memmap schema` | JSON Schema of the config | | `memmap guide` | Print the agent guide, for agents using an installed copy | @@ -148,6 +175,7 @@ scripts and agents. - [AGENTS.md](AGENTS.md): task guide, recipes and the full diagnostic list. `memmap guide` prints it. - [docs/from-elf.md](docs/from-elf.md): drawing a linked firmware image, its map readers and limits. +- [docs/from-partitions.md](docs/from-partitions.md): drawing an ESP-IDF partition table. - [Examples](#examples): one config per feature set, below. ## Examples @@ -204,6 +232,13 @@ regenerate those too: MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py tests/test_linker.py ``` +`from-partitions` is tested on real ESP-IDF builds, where the `partition-table.bin` the build +wrote checks how memmap places the CSV. After an intended change to it, review and regenerate: + +```bash +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_partitions.py +``` + [docs/from-elf.md](docs/from-elf.md#testing) explains the approach and shows the drawings. Text is measured with a built-in Helvetica/Arial width table, so wrapping is slightly approximate diff --git a/docs/from-partitions.md b/docs/from-partitions.md new file mode 100644 index 0000000..ea40878 --- /dev/null +++ b/docs/from-partitions.md @@ -0,0 +1,166 @@ +# Drawing an ESP-IDF partition table + +`memmap from-partitions` writes a config from an ESP-IDF partition table: one flash column with +the bootloader, the partition table, every partition and the free space between them. + +```bash +memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig +``` + +![The flash of an ESP32 HomeKit project, drawn by from-partitions](../tests/golden/partitions/esp32_homekit_custom_type.svg) + +- [What it reads](#what-it-reads) +- [Where the settings come from](#where-the-settings-come-from) +- [What is drawn](#what-is-drawn) +- [Checks](#checks) +- [Limits](#limits) +- [Testing](#testing) + +## What it reads + +The table is either of the files ESP-IDF works from, told apart by their content: + +| File | Where | Offsets | +| --- | --- | --- | +| Partition table CSV | the project, named by `CONFIG_PARTITION_TABLE_FILENAME` | worked out by memmap, as the build does | +| `partition-table.bin` | `build/partition_table/` | as the build wrote them | + +A CSV has the columns `Name, Type, SubType, Offset, Size, Flags`, and memmap reads it the way +ESP-IDF's `gen_esp32part.py` does: + +- Lines starting with `#` and blank lines are skipped. A header line without `#` is an error, + as it is for the build. +- Numbers are Python integers, so `0x9000`, `36864`, `0o110000` and `0x9_000` all work, with an + optional `K` or `M` suffix in either case. +- Types are `app`, `data`, `bootloader`, `partition_table` or a number. Subtypes are ESP-IDF's + names, such as `ota_0`, `nvs` or `spiffs`, or a number. A subtype name that a component + registers is kept as written. +- A blank offset follows the partition above it, aligned to 64K for an app and to 4K for + anything else. The first one starts after the partition table. Partitions with offsets must + be listed in address order, as the build places them in file order. +- A negative size gives the offset the partition ends at: `-0x400000` runs to the end of 4 MB. +- `bootloader` and `partition_table` entries with the `primary` subtype sit at the bootloader + and partition table offsets, whatever their offset and size columns say. +- Flags are `encrypted` and `readonly`, separated by `:`. +- A byte order mark selects UTF-8, UTF-16 or UTF-32. + +A binary table is a list of 32-byte entries with an MD5 entry and `0xFF` padding. memmap checks +the MD5 and reads the entries as they are. + +## Where the settings come from + +The table gives the partitions, but not how big the flash is, where the partition table sits or +where the ROM loads the bootloader from. memmap takes each from the options, else from the +`sdkconfig` given with `--sdkconfig`, else from a default: + +| Setting | Option | sdkconfig | Default | +| --- | --- | --- | --- | +| Flash size | `--flash-size 4M` | `CONFIG_ESPTOOLPY_FLASHSIZE` | none: memmap stops and says what to give | +| Partition table offset | `--table-offset 0x8000` | `CONFIG_PARTITION_TABLE_OFFSET` | 0x8000 | +| Bootloader offset | `--bootloader-offset 0x1000` | `CONFIG_BOOTLOADER_OFFSET_IN_FLASH`, else the chip in `CONFIG_IDF_TARGET` | not known | +| Recovery bootloader | | `CONFIG_BOOTLOADER_RECOVERY_OFFSET` | none | +| Secure boot V1 | | `CONFIG_SECURE_BOOT_V1_ENABLED` | off | + +The bootloader starts at 0x1000 on the ESP32 and ESP32-S2, at 0x2000 on the ESP32-C5, ESP32-H4, +ESP32-P4 and ESP32-S31, and at 0 on the other chips. A binary table with `primary` bootloader +or partition table entries gives those offsets itself; an option or sdkconfig that says +otherwise belongs to another build, and memmap stops. The header comments of the config say +which source gave each setting. + +Projects without an sdkconfig, such as Arduino-ESP32 or PlatformIO ones, give the flash size +and, for a drawing that places the bootloader, `--bootloader-offset`. + +When the flash size is missing and an `sdkconfig` lies next to the table, the hint names it. + +## What is drawn + +One region, `flash`, from 0 to the flash size. Offsets are positions in the flash chip, not the +addresses the CPU reads the flash through. Its blocks, in address order: + +- `Reserved` below the bootloader, on a chip whose bootloader does not start at 0. +- `Bootloader`, from the bootloader offset up to the partition table. Without a known offset it + starts at 0, with a note saying it starts at 0x0, 0x1000 or 0x2000 by chip. +- `Partition table`, one 4K sector. +- Each partition, named as in the table, with a note giving its type, subtype and flags, such as + `app, ota_0` or `data, nvs_keys, encrypted`. A subtype left blank in a CSV is not shown, as + its value depends on the ESP-IDF version. +- `Free` for the space between them, with its size. + +A table that has its own `primary` bootloader or partition table entry is drawn with that entry +in place of the generated block. The region has 4K `sectors`, so `check` reports a partition +edge inside an erase sector as W100; the tick marks are turned off in `style`. The column title +gives how much of the flash the bootloader, the table and the partitions take. + +## Checks + +memmap stops, writing nothing, when the table cannot be drawn: + +- the CSV cannot be read: an unknown type or flag, a blank type or size, a bad number; +- a partition starts before the end of the partition table, or overlaps one listed before it; +- a partition runs past the end of the flash, or has size 0; +- the partition table offset is 0, is not a multiple of 0x1000, or runs past the end of the + flash, or the bootloader is not below it; +- a binary table's own bootloader or partition table entry disagrees with the options or the + sdkconfig. + +The message names the CSV line, and the hint says what to change. With `--json` it is E090. + +A CSV that can be drawn but breaks a rule of ESP-IDF's own table check gets W113, which +`--strict` turns into a failure: + +- an app starts off a 64K boundary, or anything else off a 4K one; +- an app's size is not a multiple of 4K, or of 64K with secure boot V1; +- a read-write NVS partition is smaller than 12K; +- `ota` or `coredump` data is marked `readonly`; +- there is more than one `otadata` or TEE `otadata` partition, or it is not 8K; +- two partitions share a name; +- a subtype name is not one ESP-IDF defines. The build accepts it only when a component + registers it, so check the spelling. + +Some of these rules are newer than others: a table that ESP-IDF 5.2 built can get W113 from a +rule added later. A binary table is not checked, as the build that wrote it checked it already. + +## Limits + +- **Environment variables.** The build expands `$NAME` and `${NAME}` in a CSV from the + environment. memmap stops at any `$` followed by a name in a partition line, and asks for + the value, so that the drawing does not depend on the shell it ran in. +- **Blank subtypes.** ESP-IDF 5 reads a blank subtype as `0x06`, undefined. Earlier versions + read it as 0, which is also `otadata`'s subtype. memmap follows ESP-IDF 5. +- **Flash size `detect`.** Older sdkconfigs can leave the size to be detected when flashing. + Give `--flash-size` then. +- **Other formats.** Only ESP-IDF tables are read. Arduino-ESP32, PlatformIO and MicroPython use + the same CSV format, so their tables work too. +- **Table length.** ESP-IDF's binary table holds at most 95 entries. memmap does not check this. + +## Testing + +`tests/test_from_partitions.py` runs on real tables, kept in `tests/fixtures/esp_partitions/` +with their sources and SHA-256 in `index.json`: + +- **Builds.** Five ESP-IDF projects that committed their partition table, their `sdkconfig` and + the `partition-table.bin` their build wrote. Between them: a partition table moved to 0x18000, + a custom partition type with a blank subtype, blank offsets, rules that newer ESP-IDF added, + a 1K partition, and bootloaders at 0x0, 0x1000 and 0x2000. The `.bin` is the ground truth: + memmap's placement of the CSV must match it entry for entry. The sdkconfig keeps only the + lines memmap reads. +- **Tables.** Two tables built into ESP-IDF, an ESP-IDF example with `bootloader` and + `partition_table` entries, and the default tables of Arduino-ESP32 and MicroPython. + +Each one is imported to a golden config in `tests/golden/partitions/`, and two builds are also +drawn to a golden SVG, as the drawing code is the same for every table. Each fixture shows +something the others do not; keep it that way. To add a table, add it to `refresh.py` in the +fixtures folder, then: + +```bash +uv run python tests/fixtures/esp_partitions/refresh.py --download +``` + +```bash +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_partitions.py +``` + +Before the fixtures were chosen, the same comparison ran on 121 ESP-IDF builds found on GitHub. +memmap placed 117 exactly as their `partition-table.bin`. Two differed only in a blank subtype, +which their older ESP-IDF read as 0. The `.bin` of the other two came from a different table +than the one committed, a build left stale. diff --git a/memmap/cli.py b/memmap/cli.py index aec7b07..8935c73 100644 --- a/memmap/cli.py +++ b/memmap/cli.py @@ -25,11 +25,14 @@ read_map, resolve_memories, ) +from .from_partitions import Settings as PartitionSettings +from .from_partitions import import_table, read_sdkconfig +from .from_partitions import settings as partition_settings from .pipeline import Result, run from .render.raster import RasterUnavailable, convert from .render.svg import render_svg from .schema import json_schema -from .units import parse_size +from .units import parse_address, parse_size # Process exit codes, documented in AGENTS.md. Success is 0. EXIT_VALIDATION = 1 @@ -355,6 +358,122 @@ def from_elf( raise Exit(EXIT_VALIDATION) +def _read_bytes(path: Path) -> bytes: + """A file's bytes, or the ImportProblem to report.""" + try: + return path.read_bytes() + except OSError as exc: + raise ImportProblem(f"cannot read {path}: {exc.strerror or exc}") from None + + +def _option_address(name: str, value: str | None) -> int | None: + """An offset option's value, or the ImportProblem naming the option.""" + try: + return parse_address(value) if value is not None else None + except ValueError as exc: + raise ImportProblem(f"bad {name}: {exc}") from None + + +def _table_settings( + flash_size: str | None, + table_offset: str | None, + bootloader_offset: str | None, + sdkconfig: Path | None, +) -> PartitionSettings: + """The settings for a partition table import, from the options and the sdkconfig.""" + try: + size = parse_size(flash_size) if flash_size is not None else None + except ValueError as exc: + raise ImportProblem(f"bad --flash-size: {exc}") from None + offset = _option_address("--table-offset", table_offset) + bootloader = _option_address("--bootloader-offset", bootloader_offset) + config = None + if sdkconfig is not None: + config = read_sdkconfig(_read_bytes(sdkconfig).decode("utf-8", "replace")) + name = sdkconfig.name if sdkconfig else "sdkconfig" + return partition_settings(size, offset, config, name, bootloader) + + +@app.command( + name="from-partitions", + epilog="\b\nExample:\n memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig", +) +@click.argument("table", type=click.Path(path_type=Path)) +@click.option( + "-o", "--output", type=click.Path(path_type=Path), help="Config to write. Default: TABLE.yaml." +) +@click.option( + "--sdkconfig", + type=click.Path(path_type=Path), + help="The project's sdkconfig: gives the flash size, the partition table offset and the " + "bootloader offset.", +) +@click.option("--flash-size", metavar="SIZE", help="Flash size, such as 4M. Overrides --sdkconfig.") +@click.option( + "--table-offset", + metavar="OFFSET", + help="Offset of the partition table. Default: from --sdkconfig, else 0x8000.", +) +@click.option( + "--bootloader-offset", + metavar="OFFSET", + help="Where the ROM loads the bootloader from: 0x1000 on the ESP32 and ESP32-S2, 0x2000 on " + "the ESP32-C5 and ESP32-P4, else 0. Default: from --sdkconfig.", +) +@click.option("--force", is_flag=True, help="Overwrite the output if it exists.") +@json_option +@strict_option +def from_partitions( + table: Path, + output: Path | None, + sdkconfig: Path | None, + flash_size: str | None, + table_offset: str | None, + bootloader_offset: str | None, + force: bool, + as_json: bool, + strict: bool, +) -> None: + """Write a config from an ESP-IDF partition table. + + TABLE is the partition table: the CSV the project names in CONFIG_PARTITION_TABLE_FILENAME, + or the partition-table.bin the build writes. The table does not say how big the flash is, + so give --sdkconfig or --flash-size. The written config is then checked like `memmap check`. + """ + try: + target = output or table.with_suffix(".yaml") + except ValueError: # a path without a name, such as . + raise _fail(f"{table} is not a partition table file", EXIT_USAGE, as_json) + if target.resolve() == table.resolve(): + raise _fail(f"output {target} would overwrite the partition table", EXIT_USAGE, as_json) + if target.exists() and not force: + raise _fail(f"{target} exists, use --force to overwrite it", EXIT_USAGE, as_json) + try: + settings = _table_settings(flash_size, table_offset, bootloader_offset, sdkconfig) + data = _read_bytes(table) + except ImportProblem as exc: + raise _fail(str(exc), EXIT_USAGE, as_json, hint=exc.hint) + near = table.parent / "sdkconfig" + suggest = str(near) if sdkconfig is None and near.is_file() else None + try: + imported = import_table(data, table.name, settings, suggest) + except ImportProblem as exc: + raise _fail(f"{table}: {exc}", EXIT_USAGE, as_json, hint=exc.hint) + except Exception as exc: # noqa: BLE001 # odd input must still exit 3 with a message + raise _fail(f"internal error: {exc}", EXIT_RENDER, as_json) + _write(target, imported.text.encode("utf-8"), as_json) + result = _load(target, (), as_json=as_json) + result.diagnostics[:0] = imported.diagnostics + ok = _passes(result, strict) + if as_json: + _print_json(ok, result, output=str(target)) + else: + _print_diagnostics(result, target) + click.echo(str(target)) + if not ok: + raise Exit(EXIT_VALIDATION) + + @app.command(epilog="\b\nExample:\n memmap schema > memmap.schema.json") def schema() -> None: """Print the JSON Schema of the config.""" diff --git a/memmap/from_elf/writer.py b/memmap/from_elf/writer.py index ab110c1..6f26ff0 100644 --- a/memmap/from_elf/writer.py +++ b/memmap/from_elf/writer.py @@ -10,7 +10,7 @@ COLUMN_WIDTH = 150 # px, room for section names such as .ARM.exidx and the usage title -def _quote(text: str) -> str: +def quote(text: str) -> str: """A YAML double-quoted scalar. JSON strings are valid YAML.""" return json.dumps(text, ensure_ascii=False) @@ -22,16 +22,16 @@ def comment_safe(text: str) -> str: def _item_fields(item: Item, bits: int) -> list[str]: """The keys of one region, as `key: value` strings.""" - fields = [f"id: {_quote(item.id)}", f"name: {_quote(item.name)}"] + fields = [f"id: {quote(item.id)}", f"name: {quote(item.name)}"] if item.free: fields.append("kind: reserved") fields += [f"start: {address_literal(item.start, bits)}", f"size: {size_literal(item.size)}"] if item.note: - fields.append(f"note: {_quote(item.note)}") + fields.append(f"note: {quote(item.note)}") return fields -def _item_lines(item: Item, bits: int, indent: str) -> list[str]: +def item_lines(item: Item, bits: int, indent: str) -> list[str]: """One region as list entry lines: a flow mapping on one line, or a block mapping with its children one per line.""" fields = _item_fields(item, bits) @@ -41,11 +41,11 @@ def _item_lines(item: Item, bits: int, indent: str) -> list[str]: lines += [f"{indent} {field}" for field in fields[1:]] lines.append(f"{indent} children:") for child in item.children: - lines += _item_lines(child, bits, indent + " ") + lines += item_lines(child, bits, indent + " ") return lines -def _usage_title(bucket: Bucket) -> str: +def usage_title(bucket: Bucket) -> str: """A column title: the region name, then how much of it the sections use.""" memory = bucket.memory assert memory is not None @@ -83,17 +83,17 @@ def _memory_lines(bits: int, buckets: list[Bucket], top: Bucket) -> list[str]: memory = bucket.memory assert memory is not None block = [ - f" - id: {_quote(memory.id)}", - f" name: {_quote(memory.name)}", + f" - id: {quote(memory.id)}", + f" name: {quote(memory.name)}", f" start: {address_literal(memory.start, bits)}", f" size: {size_literal(memory.size)}", ] if bucket.items: block.append(" children:") for item in bucket.items: - block += _item_lines(item, bits, " ") + block += item_lines(item, bits, " ") entries.append((memory.start, block)) - entries += [(item.start, _item_lines(item, bits, " ")) for item in top.items] + entries += [(item.start, item_lines(item, bits, " ")) for item in top.items] for _, block in sorted(entries, key=lambda entry: entry[0]): lines += block return lines @@ -109,9 +109,9 @@ def _view_lines(buckets: list[Bucket], top: Bucket) -> list[str]: for bucket in drawn: memory = bucket.memory assert memory is not None - title = _quote(_usage_title(bucket)) + title = quote(usage_title(bucket)) lines.append( - f" - {{ id: {_quote(memory.id + '_col')}, root: {_quote(memory.id)}, title: {title}, " + f" - {{ id: {quote(memory.id + '_col')}, root: {quote(memory.id)}, title: {title}, " f"width: {COLUMN_WIDTH} }}" ) else: # no regions, or sections outside them: show the whole address space @@ -119,16 +119,16 @@ def _view_lines(buckets: list[Bucket], top: Bucket) -> list[str]: for bucket in drawn: memory = bucket.memory assert memory is not None - title = _quote(_usage_title(bucket)) - expand.append(f" - {{ region: {_quote(memory.id)}, title: {title} }}") + title = quote(usage_title(bucket)) + expand.append(f" - {{ region: {quote(memory.id)}, title: {title} }}") empty = [bucket.memory for bucket in buckets if not bucket.drawn and bucket.memory] if empty: names = ", ".join(comment_safe(memory.name) for memory in empty) lines.append(f" # No section lies in {names}. To draw one, add a column with its id as root.") for region, title in _expansions(buckets, top): # a column of its own, for its width - root, column = _quote(region), _quote(f"{region}_col") + root, column = quote(region), quote(f"{region}_col") lines.append(f" - {{ id: {column}, root: {root}, width: {COLUMN_WIDTH} }}") - expand.append(f" - {{ region: {root}, to: {column}, title: {_quote(title)} }}") + expand.append(f" - {{ region: {root}, to: {column}, title: {quote(title)} }}") if expand: lines += [" expand:", *expand] return lines @@ -154,7 +154,7 @@ def write_yaml( "version: 1", "", "figure:", - f" title: {_quote(f'{source} memory map')}", + f" title: {quote(f'{source} memory map')}", "", *_memory_lines(bits, buckets, top), "", diff --git a/memmap/from_partitions/__init__.py b/memmap/from_partitions/__init__.py new file mode 100644 index 0000000..30486b0 --- /dev/null +++ b/memmap/from_partitions/__init__.py @@ -0,0 +1,11 @@ +"""`memmap from-partitions`: write a config from a flash partition table. + +- `esp_idf` reads an ESP-IDF partition table, CSV or binary, and places its partitions. +- `sdkconfig` reads the flash size and offsets from an ESP-IDF project's sdkconfig. +- `importer` lays the partitions out in the flash, and `writer` writes the YAML. +""" + +from .importer import Settings, import_table, settings +from .sdkconfig import SdkConfig, read_sdkconfig + +__all__ = ["SdkConfig", "Settings", "import_table", "read_sdkconfig", "settings"] diff --git a/memmap/from_partitions/esp_idf.py b/memmap/from_partitions/esp_idf.py new file mode 100644 index 0000000..5b1b7b0 --- /dev/null +++ b/memmap/from_partitions/esp_idf.py @@ -0,0 +1,322 @@ +"""Read an ESP-IDF partition table: the CSV a project writes, or the binary the build makes of it. + +Placement follows ESP-IDF's gen_esp32part.py, so that a CSV gives the offsets the build would +write into the binary table: a blank offset follows the previous partition, aligned to 64K for +an app and to 4K for anything else, starting after the partition table. Offsets are positions in +the flash chip, not CPU addresses. +""" + +from __future__ import annotations + +import codecs +import hashlib +import re +import struct +from dataclasses import dataclass + +from ..from_elf.regions import ImportProblem + +TABLE_SIZE = 0x1000 # the partition table takes one flash sector +SECTOR = 0x1000 # SPI flash erase sector +APP_ALIGN = 0x10000 # an app partition starts on a 64K boundary, as the MMU maps 64K pages + +APP, DATA, BOOTLOADER, PARTITION_TABLE = 0x00, 0x01, 0x02, 0x03 +TYPES = {"app": APP, "data": DATA, "bootloader": BOOTLOADER, "partition_table": PARTITION_TABLE} +PRIMARY = 0x00 # the subtype of the bootloader and partition table the ROM boots from +SUBTYPES: dict[int, dict[str, int]] = { + BOOTLOADER: {"primary": 0x00, "ota": 0x01, "recovery": 0x02}, + PARTITION_TABLE: {"primary": 0x00, "ota": 0x01}, + APP: { + "factory": 0x00, + "test": 0x20, + **{f"ota_{slot}": 0x10 + slot for slot in range(16)}, + **{f"tee_{slot}": 0x30 + slot for slot in range(2)}, + }, + DATA: { + "ota": 0x00, + "phy": 0x01, + "nvs": 0x02, + "coredump": 0x03, + "nvs_keys": 0x04, + "efuse": 0x05, + "undefined": 0x06, + "esphttpd": 0x80, + "fat": 0x81, + "spiffs": 0x82, + "littlefs": 0x83, + "tee_ota": 0x90, + }, +} +UNDEFINED = SUBTYPES[DATA]["undefined"] # what a blank subtype means +FLAGS = {"encrypted": 0, "readonly": 1} # flag name: bit in the binary entry + +_ENTRY = struct.Struct("<2sBBLL16sL") # magic, type, subtype, offset, size, name, flags +_MAGIC = b"\xaa\x50" +_MD5_MAGIC = b"\xeb\xeb" +_END = b"\xff" * _ENTRY.size +_BOMS = ( + (codecs.BOM_UTF32_LE, "utf-32"), # before UTF-16, whose LE mark it starts with + (codecs.BOM_UTF32_BE, "utf-32"), + (codecs.BOM_UTF8, "utf-8-sig"), + (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16"), +) +# The build expands environment variables, as os.path.expandvars does, before reading a line. +_VARIABLE = re.compile(r"(? int: + """The first offset after the partition.""" + return self.offset + self.size + + @property + def type_name(self) -> str: + """`app`, `data` and so on, or the number in hex for a custom type.""" + return next((name for name, value in TYPES.items() if value == self.type), f"0x{self.type:02x}") + + @property + def subtype_name(self) -> str: + """`ota_0`, `nvs` and so on, or the number in hex when the type has no such name.""" + names = SUBTYPES.get(self.type, {}) + if self.subtype is None: + return self.subtype_text or "" + return next((name for name, value in names.items() if value == self.subtype), f"0x{self.subtype:02x}") + + def is_primary(self, ptype: int) -> bool: + """Whether this is the primary bootloader or partition table, which the ROM finds at a + fixed offset rather than through the table.""" + return self.type == ptype and self.subtype == PRIMARY + + @property + def where(self) -> str: + """How messages name the partition: `line 4 (nvs)`, or just the name for a binary.""" + return f"line {self.line} ({self.name})" if self.line is not None else self.name + + +def is_binary(data: bytes) -> bool: + """Whether `data` is a binary table: it starts with an entry's magic bytes.""" + return data[:2] == _MAGIC + + +def alignment(ptype: int) -> int: + """The offset an entry of this type must be a multiple of.""" + return APP_ALIGN if ptype == APP else SECTOR + + +def read_binary(data: bytes) -> list[Partition]: + """The entries of a binary table, as the build writes it: 32-byte entries, an MD5 entry, + then 0xFF padding.""" + md5 = hashlib.md5(usedforsecurity=False) + partitions: list[Partition] = [] + for number, at in enumerate(range(0, len(data), _ENTRY.size), 1): + chunk = data[at : at + _ENTRY.size] + if len(chunk) != _ENTRY.size: + raise ImportProblem("the binary table ends inside an entry", hint="rebuild the table") + if chunk == _END: + return partitions + if chunk[:2] == _MD5_MAGIC: + if chunk[16:] != md5.digest(): + raise ImportProblem( + "the binary table's MD5 checksum does not match its entries", + hint="the file is damaged; rebuild the table", + ) + continue + md5.update(chunk) + magic, ptype, subtype, offset, size, raw_name, flag_bits = _ENTRY.unpack(chunk) + if magic != _MAGIC: + raise ImportProblem(f"entry {number} of the binary table is not a partition entry") + name = raw_name.split(b"\0", 1)[0].decode("utf-8", "replace") + flags = tuple(flag for flag, bit in FLAGS.items() if flag_bits & (1 << bit)) + partitions.append(Partition(name, ptype, subtype, offset, size, flags)) + raise ImportProblem("the binary table has no end marker", hint="rebuild the table") + + +def decode(data: bytes) -> str: + """The text of a CSV table, honouring a byte order mark as the build does.""" + encoding = next((name for bom, name in _BOMS if data.startswith(bom)), "utf-8") + try: + return data.decode(encoding) + except UnicodeDecodeError: + raise ImportProblem("the file is neither UTF-8 text nor a binary partition table") from None + + +def parse_int(text: str, keywords: dict[str, int] | None = None) -> int: + """A number as gen_esp32part.py reads one: Python's int(text, 0), so 0x and 0o prefixes and + underscores work, with K and M suffixes, else one of `keywords`. Raises ValueError.""" + for letter, factor in (("k", 1024), ("m", 1024 * 1024)): + if text.lower().endswith(letter): + return parse_int(text[:-1], keywords) * factor + try: + return int(text, 0) + except ValueError: + if keywords and text.lower() in keywords: + return keywords[text.lower()] + raise + + +@dataclass +class _Row: + """A CSV line before placement: the offset may be blank, and a negative size gives the + offset the partition ends at.""" + + partition: Partition + offset: int | None + size: int + + +def _number(text: str, line: int, column: str) -> int: + """An offset or size field as a number, or the error the build would stop with.""" + try: + return parse_int(text) + except ValueError: + raise ImportProblem(f"line {line}: bad {column} {text!r}", hint=CSV_HINT) from None + + +def _row( + line: str, number: int, table_offset: int, bootloader: int | None, recovery: int | None +) -> _Row: + """One CSV line. Missing trailing fields are blank.""" + name, ptype_text, subtype_text, offset_text, size_text, flags_text = [ + part.strip() for part in (line + ",,,,,").split(",")[:6] + ] + if not ptype_text: + raise ImportProblem(f"line {number}: the type is blank", hint=CSV_HINT) + try: + ptype = parse_int(ptype_text, TYPES) + except ValueError: + hint = f"use {', '.join(TYPES)} or a number" + if ptype_text.strip('"').lower() == "type": + hint = "start the header line with #: every other line is read as a partition" + raise ImportProblem(f"line {number}: unknown type {ptype_text!r}", hint=hint) from None + if not 0 <= ptype <= 0xFF: + raise ImportProblem(f"line {number}: type {ptype_text} is not between 0 and 0xFF") + if not subtype_text: + if ptype == APP: + raise ImportProblem(f"line {number}: an app partition needs a subtype", hint=CSV_HINT) + subtype: int | None = UNDEFINED + else: + try: + subtype = parse_int(subtype_text, SUBTYPES.get(ptype, {})) + except ValueError: + if not re.fullmatch(r"[A-Za-z_]\w*", subtype_text): + raise ImportProblem(f"line {number}: bad subtype {subtype_text!r}", hint=CSV_HINT) from None + subtype = None # a subtype a component registers: the build knows it, memmap does not + if subtype is not None and not 0 <= subtype <= 0xFF: + raise ImportProblem( + f"line {number}: subtype {subtype_text} is not between 0 and 0xFF", hint=CSV_HINT + ) + flags = tuple(flag for flag in flags_text.split(":") if flag) + for flag in flags: + if flag not in FLAGS: + raise ImportProblem( + f"line {number}: unknown flag {flag!r}", + hint="the flags are encrypted and readonly, separated by a colon", + ) + partition = Partition(name, ptype, subtype, 0, 0, flags, number, subtype_text) + + def needs(value: int | None, what: str, setting: str, option: str = "") -> int: + """The bootloader offset an entry is placed at, or the error saying where it comes from.""" + if value is None: + raise ImportProblem( + f"line {number}: a {what} entry is placed at the {what} offset, which is not known", + hint=f"pass {option}the project's sdkconfig with --sdkconfig, which gives it as " + f"{setting}", + ) + return value + + offset: int | None = None + if ptype == BOOTLOADER and subtype == PRIMARY: + offset = needs(bootloader, "bootloader", BOOTLOADER_SETTING, BOOTLOADER_OPTION) + elif ptype == BOOTLOADER and subtype == SUBTYPES[BOOTLOADER]["recovery"]: + offset = needs(recovery, "recovery bootloader", "CONFIG_BOOTLOADER_RECOVERY_OFFSET") + elif ptype == PARTITION_TABLE and subtype == PRIMARY: + offset = table_offset + elif offset_text: + offset = _number(offset_text, number, "offset") + if ptype == BOOTLOADER: # every bootloader slot is as big as the space below the table + size = table_offset - needs(bootloader, "bootloader", BOOTLOADER_SETTING, BOOTLOADER_OPTION) + elif ptype == PARTITION_TABLE: + size = TABLE_SIZE + elif size_text: + size = _number(size_text, number, "size") + else: + raise ImportProblem(f"line {number}: the size is blank", hint=CSV_HINT) + return _Row(partition, offset, size) + + +def _place(rows: list[_Row], table_offset: int) -> None: + """Fill in blank offsets and negative sizes, as the build does: each partition follows the + one above it in the file, so listed offsets must go up.""" + last_end = table_offset + TABLE_SIZE + previous: Partition | None = None + for row in rows: + partition = row.partition + if partition.is_primary(BOOTLOADER) or partition.is_primary(PARTITION_TABLE): + assert row.offset is not None + partition.offset, partition.size = row.offset, row.size + continue # at fixed offsets, below the other partitions + if row.offset is not None and row.offset < last_end: + if previous is None: + where = f"the partition table ends at 0x{last_end:x}" + hint = f"start it at 0x{last_end:x} or higher, or leave its offset blank" + elif row.offset >= previous.offset: + where = f"{previous.name}, listed before it, runs to 0x{last_end - 1:x}" + hint = f"start it at 0x{last_end:x} or higher, or make {previous.name} smaller" + else: + where = f"it is listed after {previous.name}, which starts higher" + hint = "list partitions in address order: the build places them in file order" + raise ImportProblem( + f"line {partition.line}: {partition.name} starts at {hex(row.offset)}, but {where}", + hint=hint, + ) + if row.offset is None: + row.offset = -(-last_end // alignment(partition.type)) * alignment(partition.type) + size = row.size if row.size >= 0 else -row.size - row.offset + if size <= 0: + raise ImportProblem( + f"line {partition.line}: {partition.name} has size {size}, which memmap cannot draw", + hint="give it a size, or leave it out of the table", + ) + partition.offset, partition.size = row.offset, size + last_end = partition.stop + previous = partition + + +def read_csv( + text: str, table_offset: int, bootloader: int | None = None, recovery: int | None = None +) -> list[Partition]: + """The entries of a CSV table, placed. `bootloader` and `recovery` are the offsets of the + bootloader slots, needed only when the table lists them.""" + rows = [] + for number, raw in enumerate(text.splitlines(), 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + variable = _VARIABLE.search(line) + if variable: + raise ImportProblem( + f"line {number}: {variable.group(0)} is an environment variable, which the build " + "expands and memmap does not", + hint="replace it with its value", + ) + rows.append(_row(line, number, table_offset, bootloader, recovery)) + _place(rows, table_offset) + return [row.partition for row in rows] diff --git a/memmap/from_partitions/importer.py b/memmap/from_partitions/importer.py new file mode 100644 index 0000000..608d304 --- /dev/null +++ b/memmap/from_partitions/importer.py @@ -0,0 +1,385 @@ +"""Turn an ESP-IDF partition table into a memmap config: one flash column holding the +bootloader, the partition table, every partition and the free space between them. + +A table says where each partition is, but not how big the flash is, where the partition table +itself sits or where the ROM loads the bootloader from. Those come from the project's sdkconfig +or from options, and the config's header says which gave each one. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass + +from ..diagnostics import Diagnostic +from ..from_elf.blocks import Bucket, Item +from ..from_elf.importer import Imported +from ..from_elf.regions import ImportProblem, MemorySpec, unique_id +from ..from_elf.sizes import approx_size, floor_size +from . import esp_idf +from .esp_idf import ( + APP, + BOOTLOADER, + DATA, + PARTITION_TABLE, + SECTOR, + TABLE_SIZE, + Partition, +) +from .sdkconfig import SdkConfig +from .writer import write_yaml + +DEFAULT_TABLE_OFFSET = 0x8000 # CONFIG_PARTITION_TABLE_OFFSET's default +OTADATA_SIZE = 0x2000 +NVS_MIN_SIZE = 0x3000 # a read-write NVS partition needs at least three sectors +BOOTLOADER_UNKNOWN_NOTE = "from 0x0, 0x1000 or 0x2000, by chip" + + +@dataclass(frozen=True) +class Setting: + """A number the drawing needs, and what gave it, for the config's header.""" + + value: int | None + origin: str + + +@dataclass(frozen=True) +class Settings: + """Where the flash, partition table and bootloader are, with their origins.""" + + flash_size: Setting + table_offset: Setting + bootloader_offset: Setting + recovery_offset: int | None = None + secure_boot_v1: bool = False + flash_size_text: str | None = None # an sdkconfig value that is not a size, for the hint + sdkconfig_name: str | None = None + + +def settings( + flash_size: int | None, + table_offset: int | None, + sdkconfig: SdkConfig | None = None, + sdkconfig_name: str = "sdkconfig", + bootloader_offset: int | None = None, +) -> Settings: + """The settings, from the options where given, else from the sdkconfig, else the defaults.""" + config = sdkconfig or SdkConfig() + if flash_size is not None: + flash = Setting(flash_size, "--flash-size") + elif config.flash_size is not None: + flash = Setting(config.flash_size, f"CONFIG_ESPTOOLPY_FLASHSIZE in {sdkconfig_name}") + else: + flash = Setting(None, "") + if table_offset is not None: + table = Setting(table_offset, "--table-offset") + elif config.table_offset is not None: + table = Setting(config.table_offset, f"CONFIG_PARTITION_TABLE_OFFSET in {sdkconfig_name}") + else: + table = Setting(DEFAULT_TABLE_OFFSET, "the default") + if bootloader_offset is not None: + bootloader = Setting(bootloader_offset, "--bootloader-offset") + elif config.bootloader_offset is not None: + bootloader = Setting(config.bootloader_offset, f"{config.bootloader_from} in {sdkconfig_name}") + else: + bootloader = Setting(None, "") + return Settings( + flash, + table, + bootloader, + recovery_offset=config.recovery_offset, + secure_boot_v1=config.secure_boot_v1, + flash_size_text=config.flash_size_text, + sdkconfig_name=sdkconfig_name if sdkconfig else None, + ) + + +def _hex(value: int) -> str: + """An offset for a message: 0x9000.""" + return f"0x{value:x}" + + +def _check_settings(settings: Settings, flash_size: int) -> None: + """Reject a partition table offset or bootloader offset the build would reject, or that + leaves the bootloader or the partition table no room in the flash.""" + table = settings.table_offset.value + assert table is not None + if table % SECTOR: + raise ImportProblem( + f"the partition table offset {_hex(table)} from {settings.table_offset.origin} is not " + "a multiple of 0x1000" + ) + bootloader = settings.bootloader_offset.value + if bootloader is not None and bootloader >= table: + raise ImportProblem( + f"the bootloader at {_hex(bootloader)} is not below the partition table at {_hex(table)}" + ) + if table == 0: + raise ImportProblem( + f"the partition table at 0x0, from {settings.table_offset.origin}, leaves no room for " + "the bootloader below it", + hint="give the offset of CONFIG_PARTITION_TABLE_OFFSET, 0x8000 by default", + ) + if table + TABLE_SIZE > flash_size: + raise ImportProblem( + f"the partition table at {_hex(table)} runs past the end of the " + f"{approx_size(flash_size)} flash", + hint="check the flash size and the partition table offset", + ) + + +def _flash_size(settings: Settings, near: str | None) -> int: + """The flash size, or the problem that says how to give it. `near` names an sdkconfig + found next to the table, which the hint then suggests.""" + if settings.flash_size.value is not None: + return settings.flash_size.value + if settings.flash_size_text is not None: + message = ( + f"{settings.sdkconfig_name} sets CONFIG_ESPTOOLPY_FLASHSIZE to " + f"{settings.flash_size_text!r}, which is not a size" + ) + hint = "pass the size of the flash chip with --flash-size, such as --flash-size 4M" + elif settings.sdkconfig_name is not None: + message = f"{settings.sdkconfig_name} does not set CONFIG_ESPTOOLPY_FLASHSIZE" + hint = "pass the size of the flash chip with --flash-size, such as --flash-size 4M" + else: + message = "the flash size is not known: the partition table does not record it" + hint = "pass the project's sdkconfig with --sdkconfig, or the size with --flash-size 4M" + if near: + hint = f"pass --sdkconfig {near}, or the size with --flash-size, such as 4M" + raise ImportProblem(message, hint=hint) + + +def _check_fit(partitions: list[Partition], table_offset: int, flash_size: int) -> None: + """Stop on what cannot be drawn: partitions that overlap each other, the partition table + or the bootloader below it, or run past the end of the flash.""" + table_end = table_offset + TABLE_SIZE + last: Partition | None = None + for partition in sorted(partitions, key=lambda p: (p.offset, p.stop)): + if partition.size <= 0: + raise ImportProblem( + f"{partition.where} has size 0, which memmap cannot draw", + hint="give it a size, or leave it out of the table", + ) + fixed = partition.is_primary(BOOTLOADER) or partition.is_primary(PARTITION_TABLE) + if not fixed and partition.offset < table_end: + raise ImportProblem( + f"{partition.where} starts at {_hex(partition.offset)}, below the end of the " + f"partition table at {_hex(table_end)}", + hint="pass the partition table offset the build uses, with --sdkconfig or " + "--table-offset", + ) + if last is not None and partition.offset < last.stop: + raise ImportProblem( + f"{partition.where} ({_hex(partition.offset)} - {_hex(partition.stop - 1)}) " + f"overlaps {last.where} ({_hex(last.offset)} - {_hex(last.stop - 1)})" + ) + if partition.stop > flash_size: + raise ImportProblem( + f"{partition.where} ends at {_hex(partition.stop - 1)}, past the end of the " + f"{approx_size(flash_size)} flash", + hint="check the flash size: CONFIG_ESPTOOLPY_FLASHSIZE, or --flash-size", + ) + last = partition + + +def _rule_warnings(partitions: list[Partition], ids: dict[int, str], secure_v1: bool) -> list[Diagnostic]: + """W113 for each rule of ESP-IDF's table check that the table breaks. The build stops on + these, but the table can still be drawn. Some rules are newer than others, so a table an + older ESP-IDF accepted can break one.""" + found: list[Diagnostic] = [] + + def warn(partition: Partition, problem: str, hint: str, rejects: str = "") -> None: + """Report one broken rule. `rejects` qualifies when the build rejects it.""" + found.append( + Diagnostic( + "W113", + f"{partition.where} {problem}, which current ESP-IDF rejects{rejects}", + hint=hint, + subject=ids[id(partition)], + ) + ) + + for partition in partitions: + align = esp_idf.alignment(partition.type) + if partition.offset % align: + fixed = -(-partition.offset // align) * align + warn( + partition, + f"starts at {_hex(partition.offset)}, not on a {approx_size(align)} boundary", + f"start it at {_hex(fixed)}, or leave its offset blank", + ) + size_align = 0x10000 if secure_v1 else SECTOR + if partition.type == APP and partition.size % size_align: + warn( + partition, + f"has size {_hex(partition.size)}, not a multiple of {approx_size(size_align)}", + f"round the size up to {_hex(-(-partition.size // size_align) * size_align)}", + ) + if partition.type == DATA and partition.subtype in (0x00, 0x03) and "readonly" in partition.flags: + warn(partition, f"is {partition.subtype_name} data marked readonly", "remove the readonly flag") + nvs = partition.type == DATA and partition.subtype == esp_idf.SUBTYPES[DATA]["nvs"] + if nvs and partition.size < NVS_MIN_SIZE and "readonly" not in partition.flags: + warn( + partition, + f"is a read-write NVS partition of {approx_size(partition.size)}, less than 12 KB", + "make it at least 0x3000, or flag it readonly", + ) + for partition in partitions: + if partition.subtype is None: + warn( + partition, + f"has subtype {partition.subtype_text!r}", + "check the spelling; a subtype a component registers is fine", + " unless a component registers it", + ) + names = Counter(partition.name for partition in partitions) + for partition in partitions: + if names[partition.name] > 1: + warn(partition, f"shares its name with {names[partition.name] - 1} more", "give each partition its own name") + for subtype, label in ((0x00, "otadata (data, ota)"), (0x90, "TEE otadata (data, tee_ota)")): + matching = [p for p in partitions if p.type == DATA and p.subtype == subtype] + for partition in matching: + if len(matching) > 1: + warn(partition, f"is one of {len(matching)} {label} partitions", "keep only one") + elif partition.size != OTADATA_SIZE: + warn(partition, f"is an {label} partition of {_hex(partition.size)} bytes", "make it 0x2000") + return found + + +def _note(partition: Partition) -> str: + """The text beside a partition: its type, subtype and flags. A subtype left blank in the + CSV is left out, as its value depends on the ESP-IDF version.""" + subtype = [] if partition.subtype_text == "" else [partition.subtype_name] + return ", ".join([partition.type_name, *subtype, *partition.flags]) + + +def _items( + partitions: list[Partition], settings: Settings, flash_size: int +) -> tuple[list[Item], dict[int, str]]: + """The blocks of the flash column, in address order with free space between them, and the + id of each partition's block by id(partition).""" + taken = {"flash"} + table_offset = settings.table_offset.value + assert table_offset is not None + bootloader = settings.bootloader_offset.value + placed: list[Item] = [] + ids: dict[int, str] = {} + boot: Item | None = None + if not any(p.is_primary(BOOTLOADER) for p in partitions): + start = bootloader if bootloader is not None else 0 + note = None if bootloader is not None else BOOTLOADER_UNKNOWN_NOTE + boot = Item( + unique_id("bootloader", taken, "bootloader"), + "Bootloader", + start, + table_offset - start, + note, + ) + placed.append(boot) + if not any(p.is_primary(PARTITION_TABLE) for p in partitions): + table_id = unique_id("partition_table", taken, "table") + placed.append(Item(table_id, "Partition table", table_offset, TABLE_SIZE)) + for partition in partitions: + item_id = unique_id(partition.name, taken, "partition") + item = Item(item_id, partition.name, partition.offset, partition.size, _note(partition)) + ids[id(partition)] = item_id + placed.append(item) + if partition.is_primary(BOOTLOADER): + boot = item + placed.sort(key=lambda item: item.start) + items: list[Item] = [] + cursor = 0 + for block in [*placed, None]: + stop = block.start if block is not None else flash_size + if stop > cursor: + if block is not None and block is boot and cursor == 0: # the ROM never reads it + name, free_id = "Reserved", unique_id("reserved", taken, "reserved") + else: + name, free_id = f"Free\n{floor_size(stop - cursor)}", unique_id("free", taken, "free") + items.append(Item(free_id, name, cursor, stop - cursor, free=True)) + if block is not None: + items.append(block) + cursor = max(cursor, block.start + block.size) + return items, ids + + +def _own_entry(setting: Setting, partitions: list[Partition], ptype: int, what: str) -> Setting: + """The offset of a binary table's own primary entry of type `ptype`, which wins over the + default. An option or sdkconfig that says otherwise belongs to another + build, so that stops the import.""" + entry = next((p for p in partitions if p.is_primary(ptype)), None) + if entry is None: + return setting + given = setting.value is not None and setting.origin != "the default" + if given and setting.value != entry.offset: + assert setting.value is not None + raise ImportProblem( + f"the table's own {what} entry is at {_hex(entry.offset)}, but {setting.origin} " + f"says {_hex(setting.value)}", + hint="check that the sdkconfig belongs to the build that wrote this table", + ) + if given: + return setting + return Setting(entry.offset, "its own entry in the table") + + +def import_table(data: bytes, source: str, settings: Settings, near_sdkconfig: str | None = None) -> Imported: + """The config for an ESP-IDF partition table, CSV or binary. `source` is the file name and + `near_sdkconfig` an sdkconfig next to it, suggested when the flash size is missing.""" + table_offset = settings.table_offset + binary = esp_idf.is_binary(data) + if binary: + partitions = esp_idf.read_binary(data) + kind = "an ESP-IDF binary partition table" + else: + assert table_offset.value is not None + partitions = esp_idf.read_csv( + esp_idf.decode(data), + table_offset.value, + settings.bootloader_offset.value, + settings.recovery_offset, + ) + kind = "an ESP-IDF partition table" + if binary: # a CSV's own entries are placed at the offsets the settings give + settings = Settings( + settings.flash_size, + _own_entry(table_offset, partitions, PARTITION_TABLE, "partition table"), + _own_entry(settings.bootloader_offset, partitions, BOOTLOADER, "bootloader"), + settings.recovery_offset, + settings.secure_boot_v1, + settings.flash_size_text, + settings.sdkconfig_name, + ) + table_offset = settings.table_offset + flash_size = _flash_size(settings, near_sdkconfig) + _check_settings(settings, flash_size) + assert table_offset.value is not None + _check_fit(partitions, table_offset.value, flash_size) + items, ids = _items(partitions, settings, flash_size) + # A binary table passed the check of the ESP-IDF version that built it, so only a CSV is + # checked here: newer versions check more, and older ones read blank subtypes differently. + diagnostics = [] if binary else _rule_warnings(partitions, ids, settings.secure_boot_v1) + used = sum(item.size for item in items if not item.free) + bucket = Bucket(MemorySpec("flash", 0, flash_size, "Flash"), items, used) + origins = _origins(settings, flash_size) + return Imported(write_yaml(source, kind, bucket, origins), diagnostics) + + +def _origins(settings: Settings, flash_size: int) -> list[str]: + """Header lines saying where the flash size and offsets came from.""" + table = settings.table_offset + bootloader = settings.bootloader_offset + assert table.value is not None + lines = [ + f"Flash size {approx_size(flash_size)}, from {settings.flash_size.origin}.", + f"Partition table at {_hex(table.value)}, from {table.origin}.", + ] + if bootloader.value is not None: + lines.append(f"Bootloader at {_hex(bootloader.value)}, from {bootloader.origin}.") + else: + lines.append( + "Bootloader offset not known, so the Bootloader block starts at 0: pass " + "--bootloader-offset or --sdkconfig to place it." + ) + return lines diff --git a/memmap/from_partitions/sdkconfig.py b/memmap/from_partitions/sdkconfig.py new file mode 100644 index 0000000..1687364 --- /dev/null +++ b/memmap/from_partitions/sdkconfig.py @@ -0,0 +1,84 @@ +"""Read the settings a partition table drawing needs from an ESP-IDF project's sdkconfig.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +_LINE = re.compile(r"^(CONFIG_[A-Za-z0-9_]+)=(.*)$", re.MULTILINE) +_FLASH_SIZE = re.compile(r"^(\d+)\s*(KB|MB)$", re.IGNORECASE) + +# Where the ROM looks for the second-stage bootloader, for sdkconfigs from before ESP-IDF wrote +# CONFIG_BOOTLOADER_OFFSET_IN_FLASH. From ESP_ROM_BOOTLOADER_OFFSET_FLASH in esp_rom; chips not +# listed use 0. +BOOTLOADER_OFFSETS = { + "esp32": 0x1000, + "esp32s2": 0x1000, + "esp32c5": 0x2000, + "esp32h4": 0x2000, + "esp32p4": 0x2000, + "esp32s31": 0x2000, +} + + +@dataclass(frozen=True) +class SdkConfig: + """The settings that place the partition table, each None when the sdkconfig lacks it. + `flash_size_text` keeps CONFIG_ESPTOOLPY_FLASHSIZE as written, for a value such as + `detect` that is not a size.""" + + flash_size: int | None = None + flash_size_text: str | None = None + table_offset: int | None = None + bootloader_offset: int | None = None + bootloader_from: str | None = None # the setting that gave bootloader_offset + recovery_offset: int | None = None + target: str | None = None + secure_boot_v1: bool = False # app sizes then have to be multiples of 64K + + +def _value(raw: str) -> str: + """A value without its quotes and escapes.""" + raw = raw.strip() + if len(raw) >= 2 and raw[0] == raw[-1] == '"': + return re.sub(r"\\(.)", r"\1", raw[1:-1]) + return raw + + +def _int(values: dict[str, str], key: str) -> int | None: + """A numeric setting, or None when it is missing or not a number.""" + try: + return int(values[key], 0) + except (KeyError, ValueError): + return None + + +def read_sdkconfig(text: str) -> SdkConfig: + """The settings in an sdkconfig's text. Lines that are not `CONFIG_...=value` are ignored.""" + values = {match.group(1): _value(match.group(2)) for match in _LINE.finditer(text)} + flash_text = values.get("CONFIG_ESPTOOLPY_FLASHSIZE") + flash_size = None + if flash_text is not None: + match = _FLASH_SIZE.match(flash_text.strip()) + if match: + factor = 1024 if match.group(2).upper() == "KB" else 1024 * 1024 + flash_size = int(match.group(1)) * factor or None + target = values.get("CONFIG_IDF_TARGET") or None + bootloader = _int(values, "CONFIG_BOOTLOADER_OFFSET_IN_FLASH") + bootloader_from = "CONFIG_BOOTLOADER_OFFSET_IN_FLASH" if bootloader is not None else None + if bootloader is None and target is not None: + bootloader = BOOTLOADER_OFFSETS.get(target, 0) + bootloader_from = f"CONFIG_IDF_TARGET {target}" + secure = values.get("CONFIG_SECURE_BOOT") == "y" + v1 = values.get("CONFIG_SECURE_BOOT_V1_ENABLED") == "y" + short = values.get("CONFIG_SECURE_BOOT_ALLOW_SHORT_APP_PARTITION") == "y" + return SdkConfig( + flash_size=flash_size, + flash_size_text=flash_text, + table_offset=_int(values, "CONFIG_PARTITION_TABLE_OFFSET"), + bootloader_offset=bootloader, + bootloader_from=bootloader_from, + recovery_offset=_int(values, "CONFIG_BOOTLOADER_RECOVERY_OFFSET") or None, + target=target, + secure_boot_v1=secure and v1 and not short, + ) diff --git a/memmap/from_partitions/writer.py b/memmap/from_partitions/writer.py new file mode 100644 index 0000000..36dd27c --- /dev/null +++ b/memmap/from_partitions/writer.py @@ -0,0 +1,60 @@ +"""Write a partition table import as config text: one flash region, one line per partition.""" + +from __future__ import annotations + +from ..from_elf.blocks import Bucket +from ..from_elf.sizes import address_literal, size_literal +from ..from_elf.writer import COLUMN_WIDTH, comment_safe, item_lines, quote, usage_title +from .esp_idf import SECTOR + +BITS = 32 # flash offsets are drawn with 32-bit addresses + + +def write_yaml(source: str, kind: str, bucket: Bucket, origins: list[str]) -> str: + """The config text. `kind` says what `source` is, and each of `origins` becomes a comment + line in the header.""" + memory = bucket.memory + assert memory is not None + lines = [ + f"# Generated by `memmap from-partitions` from {comment_safe(source)}, {kind}.", + "# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate", + "# blocks freely.", + *(f"# {comment_safe(origin)}" for origin in origins), + "version: 1", + "", + "figure:", + f" title: {quote(f'{source} flash layout')}", + "", + "memory:", + " regions:", + f" - id: {quote(memory.id)}", + f" name: {quote(memory.name)}", + f" start: {address_literal(memory.start, BITS)}", + f" size: {size_literal(memory.size)}", + ] + if memory.size % SECTOR == 0: + count = memory.size // SECTOR + lines.append( + f" sectors: {{ size: {size_literal(SECTOR)}, count: {count} }} # checks alignment (W100)" + ) + lines.append(" children:") + for item in bucket.items: + lines += item_lines(item, BITS, " ") + title = quote(usage_title(bucket)) + lines += [ + "", + "view:", + " defaults:", + " labels: { size: true }", + " columns:", + ( + f" - {{ id: \"flash_col\", root: {quote(memory.id)}, title: {title}, " + f"width: {COLUMN_WIDTH} }}" + ), + "", + "style:", + " reserved: { show_label: true } # names the Free blocks", + " sector_ticks: false # 4K sectors are too many to mark", + "", + ] + return "\n".join(lines) diff --git a/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partition-table.bin b/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partition-table.bin new file mode 100644 index 0000000000000000000000000000000000000000..b39987b9254ec6b9770d3ae299be83eef56a0515 GIT binary patch literal 3072 zcmeH@K?=e!5Ji8*1Js@3#v7>M&V^UV4bl)JEkY}85DP9mh#R*(jaP8py(>khl?LMt zgc%6Q%p;k}tHcVp0+<1X(Nde-Qs$x(K@SLk2uSyZt9J+D#Ar=`%&N7rszdEgJiv)~ z2<+lgCTa5XenY4QM$LUmV|Y;s41lrYbB_X_`4;Vb8>MZQm-2f~;(bKKsSDNR)|B2S p>iXqHygVX&-iNot<1($|vj;h018jf|umLu}2G{@_U;}?{;0@JH5WD~Y literal 0 HcmV?d00001 diff --git a/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partitions.csv b/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partitions.csv new file mode 100644 index 0000000..f42e062 --- /dev/null +++ b/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/partitions.csv @@ -0,0 +1,10 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: Firmware partition offset needs to be 64K aligned, initial 36K (9 sectors) are reserved for bootloader and partition table +sec_cert, 0x3F, ,0xd000, 0x3000, , # Never mark this as an encrypted partition +nvs, data, nvs, 0x10000, 0x6000, +otadata, data, ota, , 0x2000 +phy_init, data, phy, , 0x1000, +ota_0, app, ota_0, 0x20000, 1600K, +ota_1, app, ota_1, , 1600K, +factory_nvs, data, nvs, 0x340000, 0x6000 +nvs_keys, data, nvs_keys,0x346000, 0x1000 diff --git a/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/sdkconfig b/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/sdkconfig new file mode 100644 index 0000000..5c1b2b7 --- /dev/null +++ b/tests/fixtures/esp_partitions/builds/esp32_homekit_custom_type/sdkconfig @@ -0,0 +1,9 @@ +# Excerpt of https://raw.githubusercontent.com/UsefulElectronics/esp32-homekit-lightbulb/2e78c6427890729625384b0307f206dd09ec6f85/sdkconfig +# Espressif IoT Development Framework (ESP-IDF) Project Configuration +CONFIG_IDF_TARGET="esp32" +CONFIG_ESPTOOLPY_FLASHSIZE="4MB" +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions_hap.csv" +CONFIG_PARTITION_TABLE_FILENAME="partitions_hap.csv" +CONFIG_PARTITION_TABLE_OFFSET=0x8000 +CONFIG_PARTITION_TABLE_MD5=y diff --git a/tests/fixtures/esp_partitions/builds/esp32_small_nvs/partition-table.bin b/tests/fixtures/esp_partitions/builds/esp32_small_nvs/partition-table.bin new file mode 100644 index 0000000000000000000000000000000000000000..dc247b3407b0ae42688467adfcf2c2118e8e0a95 GIT binary patch literal 3072 zcmZ1#z{tcffq{WRfq@~ftQg2Z1*-xW85lMI#S|DA@=Fp^5=#^Tc9WtLvcxdQDS;3 ziV;Bjf%5hY3_ww+Jc@%ABV;RHzy6N`Qh%*K{bYG@*R+YxPTW?leTO1B${7uT b(GVC7fzc2c4S~@R7!85Z5Eu=CAszw%ZvPT$ literal 0 HcmV?d00001 diff --git a/tests/fixtures/esp_partitions/builds/esp32_small_nvs/partitions.csv b/tests/fixtures/esp_partitions/builds/esp32_small_nvs/partitions.csv new file mode 100644 index 0000000..e0fc74e --- /dev/null +++ b/tests/fixtures/esp_partitions/builds/esp32_small_nvs/partitions.csv @@ -0,0 +1,12 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +nvs, data, nvs, , 0x2000, +otadata, data, ota, , 0x2000, +phy_init, data, phy, , 0x1000, + +ota_0, app, ota_0, , 1920K, +ota_1, app, ota_1, , 1920K, +#storage, data, spiffs, , 0xF0000, +storage0, data, spiffs, , 0x20000, +storage1, data, spiffs, , 0x3000, +storage2, data, spiffs, , 0x3000, diff --git a/tests/fixtures/esp_partitions/builds/esp32_small_nvs/sdkconfig b/tests/fixtures/esp_partitions/builds/esp32_small_nvs/sdkconfig new file mode 100644 index 0000000..bb6a1c2 --- /dev/null +++ b/tests/fixtures/esp_partitions/builds/esp32_small_nvs/sdkconfig @@ -0,0 +1,12 @@ +# Excerpt of https://raw.githubusercontent.com/majorBien/Extremum---Beta/0460770d954b62d80888a4d9843d583cfdcdc39d/sdkconfig +# Espressif IoT Development Framework (ESP-IDF) 5.2.1 Project Configuration +CONFIG_IDF_TARGET="esp32" +CONFIG_IDF_INIT_VERSION="5.2.1" +CONFIG_BOOTLOADER_OFFSET_IN_FLASH=0x1000 +CONFIG_SECURE_BOOT_V1_SUPPORTED=y +CONFIG_ESPTOOLPY_FLASHSIZE="4MB" +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_OFFSET=0x8000 +CONFIG_PARTITION_TABLE_MD5=y diff --git a/tests/fixtures/esp_partitions/builds/esp32_table_at_0x18000/partition-table.bin b/tests/fixtures/esp_partitions/builds/esp32_table_at_0x18000/partition-table.bin new file mode 100644 index 0000000000000000000000000000000000000000..10d54b6da635291e6eda6914ebfd3055a67cae9d GIT binary patch literal 3072 zcmZ1#z{tcffsuhBfq@~fte62EtO{UcWcUD-7hqs0$f%6Z%*!l6R{XWMFV$V5mxpF9s?|Oix890n0xC%Cmsw(~?n4XJB~!`acTT z)NiqWx|G26Ba(WpOSONrp-7H$MnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1n`Cc07>U% Ai2wiq literal 0 HcmV?d00001 diff --git a/tests/fixtures/esp_partitions/builds/esp32h2_zigbee/partitions.csv b/tests/fixtures/esp_partitions/builds/esp32h2_zigbee/partitions.csv new file mode 100644 index 0000000..d8dcfcd --- /dev/null +++ b/tests/fixtures/esp_partitions/builds/esp32h2_zigbee/partitions.csv @@ -0,0 +1,7 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 550K, +zb_storage, data, fat, 0x9a000, 16K, +zb_fct, data, fat, 0x9e000, 1K, diff --git a/tests/fixtures/esp_partitions/builds/esp32h2_zigbee/sdkconfig b/tests/fixtures/esp_partitions/builds/esp32h2_zigbee/sdkconfig new file mode 100644 index 0000000..b47da19 --- /dev/null +++ b/tests/fixtures/esp_partitions/builds/esp32h2_zigbee/sdkconfig @@ -0,0 +1,13 @@ +# Excerpt of https://raw.githubusercontent.com/aimpertro/esp-zigbee-plant-monitor/45b69c3edd0149132c4f37c57e90d63f68fc27bf/sdkconfig +# Espressif IoT Development Framework (ESP-IDF) 5.1.1 Project Configuration +CONFIG_IDF_TARGET="esp32h2" +CONFIG_BOOTLOADER_OFFSET_IN_FLASH=0x0 +CONFIG_SECURE_BOOT_V2_RSA_SUPPORTED=y +CONFIG_SECURE_BOOT_V2_ECC_SUPPORTED=y +CONFIG_SECURE_BOOT_V2_PREFERRED=y +CONFIG_ESPTOOLPY_FLASHSIZE="2MB" +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_FILENAME="partitions.csv" +CONFIG_PARTITION_TABLE_OFFSET=0x8000 +CONFIG_PARTITION_TABLE_MD5=y diff --git a/tests/fixtures/esp_partitions/builds/esp32p4_singleapp/partition-table.bin b/tests/fixtures/esp_partitions/builds/esp32p4_singleapp/partition-table.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8fa03b4b3536b1f4d0def4c1fed550e8fc2acc7 GIT binary patch literal 3072 zcmZ1#z{tcffq{V`fq@~ftQg2Z1*-xW85uqR#RM1_3Nk9=GxIV_kX5V-0Pz^X3epmj zOY(~ dict[str, object]: + """A build: the table (None for a table built into ESP-IDF), sdkconfig and .bin, all in + `folder` of `repo` at commit `sha`.""" + base = f"{GITHUB}{repo}/{sha}/{folder + '/' if folder else ''}" + files = {"sdkconfig": base + "sdkconfig", "partition-table.bin": base + "build/partition_table/partition-table.bin"} + if table is not None: + files["partitions.csv"] = base + table + return {"about": about, "files": files} + + +# name: what it shows, and where each file comes from. +BUILDS: dict[str, dict[str, object]] = { + "esp32_table_at_0x18000": _build( + "energietransitie/needforheat-p1-reader-firmware", "1798396832ab69dc3f3978e237e3f7a1dc9abbf5", + "", None, "ESP32: the built-in single app table, moved up by CONFIG_PARTITION_TABLE_OFFSET", + ), + "esp32_homekit_custom_type": _build( + "UsefulElectronics/esp32-homekit-lightbulb", "2e78c6427890729625384b0307f206dd09ec6f85", + "", "partitions_hap.csv", + "ESP32, older ESP-IDF: a custom type with a blank subtype, then 0 and now 0x06", + ), + "esp32_small_nvs": _build( + "majorBien/Extremum---Beta", "0460770d954b62d80888a4d9843d583cfdcdc39d", "", "partitions.csv", + "ESP32, ESP-IDF 5.2: blank offsets, an 8 KB read-write NVS that ESP-IDF now rejects", + ), + "esp32h2_zigbee": _build( + "aimpertro/esp-zigbee-plant-monitor", "45b69c3edd0149132c4f37c57e90d63f68fc27bf", "", + "partitions.csv", + "ESP32-H2: an app size that is not a multiple of 4K, a 1K partition off a sector edge", + ), + "esp32p4_singleapp": _build( + "serfereanyang-create/my_toolkit", "1eaae36df31fe0c2bb7e3c035dbeff371e56e5dd", "hello_p4", + None, "ESP32-P4, ESP-IDF 6.0: the bootloader at 0x2000", + ), +} + +# name: where the table comes from, what it shows and the flash size to draw it in. +TABLES: dict[str, dict[str, str]] = { + **{ + f"idf_{name}": { + "url": f"{IDF}components/partition_table/{name}.csv", + "about": "built into ESP-IDF", + "flash_size": "4M", + } + for name in ("partitions_singleapp", "partitions_two_ota") + }, + "idf_ota_efuse_emul": { + "url": f"{IDF}examples/system/ota/partitions_ota/test/partitions_efuse_emul_2.csv", + "about": "ESP-IDF example: bootloader and partition_table entries, an encrypted partition", + "flash_size": "4M", + }, + "arduino_default": { + "url": f"{ARDUINO}default.csv", + "about": "Arduino-ESP32 default: explicit offsets, no spaces after some commas", + "flash_size": "4M", + }, + "micropython_4MiBplus": { + "url": f"{MICROPYTHON}partitions-4MiBplus.csv", + "about": "MicroPython: blank offsets, $IDF_PATH in a comment, lowercase k sizes", + "flash_size": "4M", + }, +} + + +def _fetch(url: str, source: Path, download: bool) -> bytes | None: + """The original file, downloaded first when allowed and missing.""" + if not source.exists() and download: + print(f"downloading {url}") + source.parent.mkdir(parents=True, exist_ok=True) + urllib.request.urlretrieve(url, source) + return source.read_bytes() if source.exists() else None + + +def trim_sdkconfig(url: str, data: bytes) -> bytes: + """The sdkconfig lines memmap reads, under a line saying where they come from.""" + lines = data.decode("utf-8", "replace").replace("\r\n", "\n").split("\n") + kept = [line for line in lines if _SDKCONFIG_KEEP.match(line)] + return "\n".join([f"# Excerpt of {url}", *kept, ""]).encode("utf-8") + + +def main() -> int: + """Refresh every fixture whose file is in build/esp_partitions/. Returns the exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--download", action="store_true", help="fetch missing files first") + args = parser.parse_args() + missing: list[str] = [] + builds = [] + for name, build in BUILDS.items(): + files = {} + for file, url in build["files"].items(): + data = _fetch(url, SOURCE_DIR / "builds" / name / file, args.download) + if data is None: + missing.append(f"builds/{name}/{file}") + continue + target = HERE / "builds" / name / file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(trim_sdkconfig(url, data) if file == "sdkconfig" else data) + files[file] = {"url": url, "sha256": hashlib.sha256(data).hexdigest()} + builds.append({"name": name, "about": build["about"], "files": files}) + print(f"wrote builds/{name}") + tables = [] + for name, table in TABLES.items(): + data = _fetch(table["url"], SOURCE_DIR / "tables" / f"{name}.csv", args.download) + if data is None: + missing.append(f"tables/{name}.csv") + continue + target = HERE / "tables" / f"{name}.csv" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + tables.append({"name": name, **table, "sha256": hashlib.sha256(data).hexdigest()}) + print(f"wrote tables/{name}.csv") + for name in missing: + print(f"skipped {name}: it is missing from build/esp_partitions/", file=sys.stderr) + if missing: + return 1 + text = json.dumps({"builds": builds, "tables": tables}, indent=2) + "\n" + (HERE / "index.json").write_text(text, encoding="utf-8", newline="\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/esp_partitions/tables/arduino_default.csv b/tests/fixtures/esp_partitions/tables/arduino_default.csv new file mode 100644 index 0000000..960469b --- /dev/null +++ b/tests/fixtures/esp_partitions/tables/arduino_default.csv @@ -0,0 +1,7 @@ +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x5000, +otadata, data, ota, 0xe000, 0x2000, +app0, app, ota_0, 0x10000, 0x140000, +app1, app, ota_1, 0x150000,0x140000, +spiffs, data, spiffs, 0x290000,0x160000, +coredump, data, coredump,0x3F0000,0x10000, diff --git a/tests/fixtures/esp_partitions/tables/idf_ota_efuse_emul.csv b/tests/fixtures/esp_partitions/tables/idf_ota_efuse_emul.csv new file mode 100644 index 0000000..4970ae9 --- /dev/null +++ b/tests/fixtures/esp_partitions/tables/idf_ota_efuse_emul.csv @@ -0,0 +1,11 @@ +# Name, Type, SubType, Offset, Size, Flags +PrimaryBTLDR, bootloader, primary, N/A, N/A, +PrimaryPrtTable, partition_table, primary, N/A, N/A, +nvs, data, nvs, , 0x6000, +nvs_key, data, nvs_keys, , 4K, +storage, data, , , 0x1000, encrypted +otadata, data, ota, , 0x2000, +phy_init, data, phy, , 0x1000, +emul_efuse, data, efuse, , 0x2000, +ota_0, app, ota_0, , 0x1B0000, +ota_1, app, ota_1, , 0x1B0000, diff --git a/tests/fixtures/esp_partitions/tables/idf_partitions_singleapp.csv b/tests/fixtures/esp_partitions/tables/idf_partitions_singleapp.csv new file mode 100644 index 0000000..5156e82 --- /dev/null +++ b/tests/fixtures/esp_partitions/tables/idf_partitions_singleapp.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +nvs, data, nvs, , 0x6000, +phy_init, data, phy, , 0x1000, +factory, app, factory, , 1M, diff --git a/tests/fixtures/esp_partitions/tables/idf_partitions_two_ota.csv b/tests/fixtures/esp_partitions/tables/idf_partitions_two_ota.csv new file mode 100644 index 0000000..7aee909 --- /dev/null +++ b/tests/fixtures/esp_partitions/tables/idf_partitions_two_ota.csv @@ -0,0 +1,8 @@ +# Name, Type, SubType, Offset, Size, Flags +# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap +nvs, data, nvs, , 0x4000, +otadata, data, ota, , 0x2000, +phy_init, data, phy, , 0x1000, +factory, app, factory, , 1M, +ota_0, app, ota_0, , 1M, +ota_1, app, ota_1, , 1M, diff --git a/tests/fixtures/esp_partitions/tables/micropython_4MiBplus.csv b/tests/fixtures/esp_partitions/tables/micropython_4MiBplus.csv new file mode 100644 index 0000000..460e5cc --- /dev/null +++ b/tests/fixtures/esp_partitions/tables/micropython_4MiBplus.csv @@ -0,0 +1,10 @@ +# This partition table is for devices with 4MiB or more of flash. +# The first 2MiB is used for bootloader, nvs, phy_init and firmware. +# The remaining flash is for the user filesystem(s). + +# Notes: the offset of the partition table itself is set in +# $IDF_PATH/components/partition_table/Kconfig.projbuild. +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 0x1F0000, diff --git a/tests/golden/partitions/arduino_default.yaml b/tests/golden/partitions/arduino_default.yaml new file mode 100644 index 0000000..f7b3fb8 --- /dev/null +++ b/tests/golden/partitions/arduino_default.yaml @@ -0,0 +1,38 @@ +# Generated by `memmap from-partitions` from arduino_default.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from --flash-size. +# Partition table at 0x8000, from the default. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "arduino_default.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 28K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 20K, note: "data, nvs" } + - { id: "otadata", name: "otadata", start: 0x0000E000, size: 8K, note: "data, ota" } + - { id: "app0", name: "app0", start: 0x00010000, size: 1280K, note: "app, ota_0" } + - { id: "app1", name: "app1", start: 0x00150000, size: 1280K, note: "app, ota_1" } + - { id: "spiffs", name: "spiffs", start: 0x00290000, size: 1408K, note: "data, spiffs" } + - { id: "coredump", name: "coredump", start: 0x003F0000, size: 64K, note: "data, coredump" } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n4 MB of 4 MB (99%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/esp32_homekit_custom_type.svg b/tests/golden/partitions/esp32_homekit_custom_type.svg new file mode 100644 index 0000000..957221e --- /dev/null +++ b/tests/golden/partitions/esp32_homekit_custom_type.svg @@ -0,0 +1,71 @@ + + + +partitions.csv flash layout +0x0000 0000 +0x0000 1000 +0x0000 8000 +0x0000 9000 +0x0000 D000 +0x0001 0000 +0x0001 6000 +0x0001 8000 +0x0001 9000 +0x0002 0000 +0x001B 0000 +0x0034 0000 +0x0034 6000 +0x0034 7000 +0x003F FFFF +Flash +3.2 MB of 4 MB (81%) + +Reserved + +Bootloader +28 KB + +Partition table +4 KB + +Free +16 KB + +sec_cert +12 KB +0x3f + +nvs +24 KB +data, nvs + +otadata +8 KB +data, ota + +phy_init +4 KB +data, phy + +Free +28 KB + +ota_0 +1600 KB +app, ota_0 + +ota_1 +1600 KB +app, ota_1 + +factory_nvs +24 KB +data, nvs + +nvs_keys +4 KB +data, nvs_keys + +Free +740 KB + diff --git a/tests/golden/partitions/esp32_homekit_custom_type.yaml b/tests/golden/partitions/esp32_homekit_custom_type.yaml new file mode 100644 index 0000000..ba265a5 --- /dev/null +++ b/tests/golden/partitions/esp32_homekit_custom_type.yaml @@ -0,0 +1,43 @@ +# Generated by `memmap from-partitions` from partitions.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from CONFIG_ESPTOOLPY_FLASHSIZE in sdkconfig. +# Partition table at 0x8000, from CONFIG_PARTITION_TABLE_OFFSET in sdkconfig. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "partitions.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 28K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "free", name: "Free\n16 KB", kind: reserved, start: 0x00009000, size: 16K } + - { id: "sec_cert", name: "sec_cert", start: 0x0000D000, size: 12K, note: "0x3f" } + - { id: "nvs", name: "nvs", start: 0x00010000, size: 24K, note: "data, nvs" } + - { id: "otadata", name: "otadata", start: 0x00016000, size: 8K, note: "data, ota" } + - { id: "phy_init", name: "phy_init", start: 0x00018000, size: 4K, note: "data, phy" } + - { id: "free_2", name: "Free\n28 KB", kind: reserved, start: 0x00019000, size: 28K } + - { id: "ota_0", name: "ota_0", start: 0x00020000, size: 1600K, note: "app, ota_0" } + - { id: "ota_1", name: "ota_1", start: 0x001B0000, size: 1600K, note: "app, ota_1" } + - { id: "factory_nvs", name: "factory_nvs", start: 0x00340000, size: 24K, note: "data, nvs" } + - { id: "nvs_keys", name: "nvs_keys", start: 0x00346000, size: 4K, note: "data, nvs_keys" } + - { id: "free_3", name: "Free\n740 KB", kind: reserved, start: 0x00347000, size: 740K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n3.2 MB of 4 MB (81%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/esp32_small_nvs.yaml b/tests/golden/partitions/esp32_small_nvs.yaml new file mode 100644 index 0000000..af86de4 --- /dev/null +++ b/tests/golden/partitions/esp32_small_nvs.yaml @@ -0,0 +1,42 @@ +# Generated by `memmap from-partitions` from partitions.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from CONFIG_ESPTOOLPY_FLASHSIZE in sdkconfig. +# Partition table at 0x8000, from CONFIG_PARTITION_TABLE_OFFSET in sdkconfig. +# Bootloader at 0x1000, from CONFIG_BOOTLOADER_OFFSET_IN_FLASH in sdkconfig. +version: 1 + +figure: + title: "partitions.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 28K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 8K, note: "data, nvs" } + - { id: "otadata", name: "otadata", start: 0x0000B000, size: 8K, note: "data, ota" } + - { id: "phy_init", name: "phy_init", start: 0x0000D000, size: 4K, note: "data, phy" } + - { id: "free", name: "Free\n8 KB", kind: reserved, start: 0x0000E000, size: 8K } + - { id: "ota_0", name: "ota_0", start: 0x00010000, size: 1920K, note: "app, ota_0" } + - { id: "ota_1", name: "ota_1", start: 0x001F0000, size: 1920K, note: "app, ota_1" } + - { id: "storage0", name: "storage0", start: 0x003D0000, size: 128K, note: "data, spiffs" } + - { id: "storage1", name: "storage1", start: 0x003F0000, size: 12K, note: "data, spiffs" } + - { id: "storage2", name: "storage2", start: 0x003F3000, size: 12K, note: "data, spiffs" } + - { id: "free_2", name: "Free\n40 KB", kind: reserved, start: 0x003F6000, size: 40K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n3.9 MB of 4 MB (99%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/esp32_table_at_0x18000.yaml b/tests/golden/partitions/esp32_table_at_0x18000.yaml new file mode 100644 index 0000000..5563b36 --- /dev/null +++ b/tests/golden/partitions/esp32_table_at_0x18000.yaml @@ -0,0 +1,36 @@ +# Generated by `memmap from-partitions` from idf_partitions_singleapp.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 2 MB, from CONFIG_ESPTOOLPY_FLASHSIZE in sdkconfig. +# Partition table at 0x18000, from CONFIG_PARTITION_TABLE_OFFSET in sdkconfig. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "idf_partitions_singleapp.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 2M + sectors: { size: 4K, count: 512 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 92K } + - { id: "partition_table", name: "Partition table", start: 0x00018000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00019000, size: 24K, note: "data, nvs" } + - { id: "phy_init", name: "phy_init", start: 0x0001F000, size: 4K, note: "data, phy" } + - { id: "factory", name: "factory", start: 0x00020000, size: 1M, note: "app, factory" } + - { id: "free", name: "Free\n896 KB", kind: reserved, start: 0x00120000, size: 896K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n1.1 MB of 2 MB (56%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/esp32h2_zigbee.svg b/tests/golden/partitions/esp32h2_zigbee.svg new file mode 100644 index 0000000..8911d4f --- /dev/null +++ b/tests/golden/partitions/esp32h2_zigbee.svg @@ -0,0 +1,49 @@ + + + +partitions.csv flash layout +0x0000 0000 +0x0000 8000 +0x0000 9000 +0x0000 F000 +0x0001 0000 +0x0009 9800 +0x0009 A000 +0x0009 E000 +0x0009 E400 +0x001F FFFF +Flash +631 KB of 2 MB (31%) + +Bootloader +32 KB + +Partition table +4 KB + +nvs +24 KB +data, nvs + +phy_init +4 KB +data, phy + +factory +550 KB +app, factory + +Free +2 KB + +zb_storage +16 KB +data, fat + +zb_fct +1 KB +data, fat + +Free +1415 KB + diff --git a/tests/golden/partitions/esp32h2_zigbee.yaml b/tests/golden/partitions/esp32h2_zigbee.yaml new file mode 100644 index 0000000..0f45929 --- /dev/null +++ b/tests/golden/partitions/esp32h2_zigbee.yaml @@ -0,0 +1,38 @@ +# Generated by `memmap from-partitions` from partitions.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 2 MB, from CONFIG_ESPTOOLPY_FLASHSIZE in sdkconfig. +# Partition table at 0x8000, from CONFIG_PARTITION_TABLE_OFFSET in sdkconfig. +# Bootloader at 0x0, from CONFIG_BOOTLOADER_OFFSET_IN_FLASH in sdkconfig. +version: 1 + +figure: + title: "partitions.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 2M + sectors: { size: 4K, count: 512 } # checks alignment (W100) + children: + - { id: "bootloader", name: "Bootloader", start: 0x00000000, size: 32K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 24K, note: "data, nvs" } + - { id: "phy_init", name: "phy_init", start: 0x0000F000, size: 4K, note: "data, phy" } + - { id: "factory", name: "factory", start: 0x00010000, size: 550K, note: "app, factory" } + - { id: "free", name: "Free\n2 KB", kind: reserved, start: 0x00099800, size: 2K } + - { id: "zb_storage", name: "zb_storage", start: 0x0009A000, size: 16K, note: "data, fat" } + - { id: "zb_fct", name: "zb_fct", start: 0x0009E000, size: 1K, note: "data, fat" } + - { id: "free_2", name: "Free\n1415 KB", kind: reserved, start: 0x0009E400, size: 1415K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n631 KB of 2 MB (31%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/esp32p4_singleapp.yaml b/tests/golden/partitions/esp32p4_singleapp.yaml new file mode 100644 index 0000000..246d832 --- /dev/null +++ b/tests/golden/partitions/esp32p4_singleapp.yaml @@ -0,0 +1,36 @@ +# Generated by `memmap from-partitions` from idf_partitions_singleapp.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 2 MB, from CONFIG_ESPTOOLPY_FLASHSIZE in sdkconfig. +# Partition table at 0x8000, from CONFIG_PARTITION_TABLE_OFFSET in sdkconfig. +# Bootloader at 0x2000, from CONFIG_BOOTLOADER_OFFSET_IN_FLASH in sdkconfig. +version: 1 + +figure: + title: "idf_partitions_singleapp.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 2M + sectors: { size: 4K, count: 512 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 8K } + - { id: "bootloader", name: "Bootloader", start: 0x00002000, size: 24K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 24K, note: "data, nvs" } + - { id: "phy_init", name: "phy_init", start: 0x0000F000, size: 4K, note: "data, phy" } + - { id: "factory", name: "factory", start: 0x00010000, size: 1M, note: "app, factory" } + - { id: "free", name: "Free\n960 KB", kind: reserved, start: 0x00110000, size: 960K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n1.1 MB of 2 MB (53%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/idf_ota_efuse_emul.yaml b/tests/golden/partitions/idf_ota_efuse_emul.yaml new file mode 100644 index 0000000..dcf4692 --- /dev/null +++ b/tests/golden/partitions/idf_ota_efuse_emul.yaml @@ -0,0 +1,42 @@ +# Generated by `memmap from-partitions` from idf_ota_efuse_emul.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from --flash-size. +# Partition table at 0x8000, from the default. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "idf_ota_efuse_emul.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "primarybtldr", name: "PrimaryBTLDR", start: 0x00001000, size: 28K, note: "bootloader, primary" } + - { id: "primaryprttable", name: "PrimaryPrtTable", start: 0x00008000, size: 4K, note: "partition_table, primary" } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 24K, note: "data, nvs" } + - { id: "nvs_key", name: "nvs_key", start: 0x0000F000, size: 4K, note: "data, nvs_keys" } + - { id: "storage", name: "storage", start: 0x00010000, size: 4K, note: "data, encrypted" } + - { id: "otadata", name: "otadata", start: 0x00011000, size: 8K, note: "data, ota" } + - { id: "phy_init", name: "phy_init", start: 0x00013000, size: 4K, note: "data, phy" } + - { id: "emul_efuse", name: "emul_efuse", start: 0x00014000, size: 8K, note: "data, efuse" } + - { id: "free", name: "Free\n40 KB", kind: reserved, start: 0x00016000, size: 40K } + - { id: "ota_0", name: "ota_0", start: 0x00020000, size: 1728K, note: "app, ota_0" } + - { id: "ota_1", name: "ota_1", start: 0x001D0000, size: 1728K, note: "app, ota_1" } + - { id: "free_2", name: "Free\n512 KB", kind: reserved, start: 0x00380000, size: 512K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n3.5 MB of 4 MB (86%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/idf_partitions_singleapp.yaml b/tests/golden/partitions/idf_partitions_singleapp.yaml new file mode 100644 index 0000000..14bd3e8 --- /dev/null +++ b/tests/golden/partitions/idf_partitions_singleapp.yaml @@ -0,0 +1,36 @@ +# Generated by `memmap from-partitions` from idf_partitions_singleapp.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from --flash-size. +# Partition table at 0x8000, from the default. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "idf_partitions_singleapp.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 28K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 24K, note: "data, nvs" } + - { id: "phy_init", name: "phy_init", start: 0x0000F000, size: 4K, note: "data, phy" } + - { id: "factory", name: "factory", start: 0x00010000, size: 1M, note: "app, factory" } + - { id: "free", name: "Free\n3008 KB", kind: reserved, start: 0x00110000, size: 3008K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n1.1 MB of 4 MB (26%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/idf_partitions_two_ota.yaml b/tests/golden/partitions/idf_partitions_two_ota.yaml new file mode 100644 index 0000000..e945477 --- /dev/null +++ b/tests/golden/partitions/idf_partitions_two_ota.yaml @@ -0,0 +1,39 @@ +# Generated by `memmap from-partitions` from idf_partitions_two_ota.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from --flash-size. +# Partition table at 0x8000, from the default. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "idf_partitions_two_ota.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 28K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 16K, note: "data, nvs" } + - { id: "otadata", name: "otadata", start: 0x0000D000, size: 8K, note: "data, ota" } + - { id: "phy_init", name: "phy_init", start: 0x0000F000, size: 4K, note: "data, phy" } + - { id: "factory", name: "factory", start: 0x00010000, size: 1M, note: "app, factory" } + - { id: "ota_0", name: "ota_0", start: 0x00110000, size: 1M, note: "app, ota_0" } + - { id: "ota_1", name: "ota_1", start: 0x00210000, size: 1M, note: "app, ota_1" } + - { id: "free", name: "Free\n960 KB", kind: reserved, start: 0x00310000, size: 960K } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n3.1 MB of 4 MB (76%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/golden/partitions/micropython_4MiBplus.yaml b/tests/golden/partitions/micropython_4MiBplus.yaml new file mode 100644 index 0000000..b2c9c87 --- /dev/null +++ b/tests/golden/partitions/micropython_4MiBplus.yaml @@ -0,0 +1,36 @@ +# Generated by `memmap from-partitions` from micropython_4MiBplus.csv, an ESP-IDF partition table. +# Offsets are positions in the flash chip, not CPU addresses. Rename, merge or annotate +# blocks freely. +# Flash size 4 MB, from --flash-size. +# Partition table at 0x8000, from the default. +# Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig. +version: 1 + +figure: + title: "micropython_4MiBplus.csv flash layout" + +memory: + regions: + - id: "flash" + name: "Flash" + start: 0x00000000 + size: 4M + sectors: { size: 4K, count: 1024 } # checks alignment (W100) + children: + - { id: "reserved", name: "Reserved", kind: reserved, start: 0x00000000, size: 4K } + - { id: "bootloader", name: "Bootloader", start: 0x00001000, size: 28K } + - { id: "partition_table", name: "Partition table", start: 0x00008000, size: 4K } + - { id: "nvs", name: "nvs", start: 0x00009000, size: 24K, note: "data, nvs" } + - { id: "phy_init", name: "phy_init", start: 0x0000F000, size: 4K, note: "data, phy" } + - { id: "factory", name: "factory", start: 0x00010000, size: 1984K, note: "app, factory" } + - { id: "free", name: "Free\n2 MB", kind: reserved, start: 0x00200000, size: 2M } + +view: + defaults: + labels: { size: true } + columns: + - { id: "flash_col", root: "flash", title: "Flash\n2 MB of 4 MB (50%)", width: 150 } + +style: + reserved: { show_label: true } # names the Free blocks + sector_ticks: false # 4K sectors are too many to mark diff --git a/tests/test_from_partitions.py b/tests/test_from_partitions.py new file mode 100644 index 0000000..6b25ae1 --- /dev/null +++ b/tests/test_from_partitions.py @@ -0,0 +1,739 @@ +"""`memmap from-partitions`: ESP-IDF partition tables, CSV and binary. + +The real tables are in tests/fixtures/esp_partitions/, see its refresh.py. For each build there, +the partition-table.bin that ESP-IDF wrote is the ground truth for placing its CSV. Configs and +two drawings are compared with tests/golden/partitions/. Regenerate after an intended change +with: + + MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_partitions.py +""" + +import hashlib +import json +import os +import struct + +import pytest +from click.testing import CliRunner +from conftest import ROOT, codes, drawn + +from memmap.cli import app +from memmap.from_elf import ImportProblem +from memmap.from_partitions import esp_idf, import_table, read_sdkconfig, settings +from memmap.from_partitions.esp_idf import read_binary, read_csv +from memmap.pipeline import run +from memmap.render.svg import render_svg + +FIXTURES = ROOT / "tests" / "fixtures" / "esp_partitions" +GOLDEN = ROOT / "tests" / "golden" / "partitions" +UPDATE = os.environ.get("MEMMAP_UPDATE_GOLDEN") == "1" +INDEX = json.loads((FIXTURES / "index.json").read_text(encoding="utf-8")) +BUILDS = [build["name"] for build in INDEX["builds"]] +TABLES = {table["name"]: table for table in INDEX["tables"]} +FOUR_MB = 4 * 1024 * 1024 +# Builds whose drawing is a golden too: the one docs/from-partitions.md shows, and one with +# warnings. The drawing code is the same for every table, so more would only repeat them. +DRAWN = {"esp32_homekit_custom_type", "esp32h2_zigbee"} +runner = CliRunner() + +# What each fixture's import reports. Everything else imports cleanly. +EXPECTED_CODES = { + "esp32_small_nvs": ["W113"], # an 8 KB read-write NVS, accepted by ESP-IDF 5.2 + "esp32h2_zigbee": ["W113", "W100", "W100"], # app size 550K, a 1K partition +} + + +def compare(golden, text): + """Check `text` against a golden file, or rewrite it with MEMMAP_UPDATE_GOLDEN=1.""" + if UPDATE: + golden.parent.mkdir(parents=True, exist_ok=True) + golden.write_text(text, encoding="utf-8", newline="\n") + assert text == golden.read_text(encoding="utf-8") + + +def invoke(*args): + return runner.invoke(app, [str(arg) for arg in args]) + + +def entries(partitions): + return [(p.name, p.type, p.subtype, p.offset, p.size, p.flags) for p in partitions] + + +def build_csv(name): + """The CSV a build used: its own, or the one built into ESP-IDF that its sdkconfig names.""" + own = FIXTURES / "builds" / name / "partitions.csv" + if own.exists(): + return own + sdkconfig = (FIXTURES / "builds" / name / "sdkconfig").read_text() + builtin = next(line for line in sdkconfig.splitlines() if "PARTITION_TABLE_FILENAME=" in line) + stem = builtin.split("=")[1].strip('"').removesuffix(".csv") + return FIXTURES / "tables" / f"idf_{stem}.csv" + + +def table_bytes(*rows): + """A binary table of (name, type, subtype, offset, size, flags) rows, as ESP-IDF writes it.""" + data = b"".join( + struct.pack("<2sBBLL16sL", b"\xaa\x50", t, s, o, size, n.encode(), f) + for n, t, s, o, size, f in rows + ) + data += b"\xeb\xeb" + b"\xff" * 14 + hashlib.md5(data).digest() + return data + b"\xff" * (0xC00 - len(data)) + + +def check_strict(text, tmp_path): + """The codes the pipeline reports for generated text, so that `== []` passes --strict.""" + file = tmp_path / "layout.yaml" + file.write_text(text, encoding="utf-8") + return codes(run(file)) + + +# The fixtures + + +def test_fixtures_are_listed(): + assert len(BUILDS) >= 5 and len(TABLES) >= 5 + for build in INDEX["builds"]: + for file in build["files"]: + assert (FIXTURES / "builds" / build["name"] / file).is_file() + for name in TABLES: + assert (FIXTURES / "tables" / f"{name}.csv").is_file() + + +@pytest.mark.parametrize("name", BUILDS) +def test_csv_places_like_the_build(name): + """memmap places the CSV where ESP-IDF's build put it in partition-table.bin.""" + folder = FIXTURES / "builds" / name + config = read_sdkconfig((folder / "sdkconfig").read_text()) + placed = read_csv( + build_csv(name).read_text(), config.table_offset, config.bootloader_offset + ) + built = read_binary((folder / "partition-table.bin").read_bytes()) + if name == "esp32_homekit_custom_type": + # ESP-IDF before 5.x read a blank subtype as 0; it is now 0x06, undefined. + assert placed[0].subtype == 0x06 and built[0].subtype == 0x00 + built[0].subtype = 0x06 + assert entries(placed) == entries(built) + + +@pytest.mark.parametrize("name", BUILDS) +@pytest.mark.parametrize("source", ["csv", "bin"]) +def test_build_imports_to_the_golden_config_and_drawing(name, source, tmp_path): + folder = FIXTURES / "builds" / name + config = read_sdkconfig((folder / "sdkconfig").read_text()) + path = build_csv(name) if source == "csv" else folder / "partition-table.bin" + imported = import_table(path.read_bytes(), path.name, settings(None, None, config)) + found = [d.code for d in imported.diagnostics] + check_strict(imported.text, tmp_path) + expected = EXPECTED_CODES.get(name, []) + if source == "bin": # the build checked the binary, so memmap only adds its sector check + expected = [code for code in expected if code != "W113"] + assert found == expected + if source == "csv": + compare(GOLDEN / f"{name}.yaml", imported.text) + if name in DRAWN: + compare(GOLDEN / f"{name}.svg", render_svg(drawn(run(tmp_path / "layout.yaml")))) + + +@pytest.mark.parametrize("name", BUILDS) +def test_csv_and_binary_draw_the_same_blocks(name, tmp_path): + folder = FIXTURES / "builds" / name + config = read_sdkconfig((folder / "sdkconfig").read_text()) + blocks = [] + for path in (build_csv(name), folder / "partition-table.bin"): + imported = import_table(path.read_bytes(), "table", settings(None, None, config)) + file = tmp_path / "layout.yaml" + file.write_text(imported.text) + result = run(file) + blocks.append([(n.id, n.start, n.end) for n in result.tree.index["flash"].children]) + assert blocks[0] == blocks[1] + + +@pytest.mark.parametrize("name", sorted(TABLES)) +def test_table_imports_to_the_golden_config(name, tmp_path): + table = TABLES[name] + path = FIXTURES / "tables" / f"{name}.csv" + flash = esp_idf.parse_int(table["flash_size"]) + # The bootloader offset of an ESP32, which the efuse example's bootloader entry needs. + config = read_sdkconfig('CONFIG_IDF_TARGET="esp32"\n') + imported = import_table(path.read_bytes(), path.name, settings(flash, None, config)) + found = [d.code for d in imported.diagnostics] + check_strict(imported.text, tmp_path) + assert found == EXPECTED_CODES.get(name, []) + compare(GOLDEN / f"{name}.yaml", imported.text) + + +def test_two_ota_table_in_detail(): + partitions = read_csv((FIXTURES / "tables" / "idf_partitions_two_ota.csv").read_text(), 0x8000) + assert [(p.name, p.type_name, p.subtype_name, p.offset, p.size) for p in partitions] == [ + ("nvs", "data", "nvs", 0x9000, 0x4000), + ("otadata", "data", "ota", 0xD000, 0x2000), + ("phy_init", "data", "phy", 0xF000, 0x1000), + ("factory", "app", "factory", 0x10000, 0x100000), + ("ota_0", "app", "ota_0", 0x110000, 0x100000), + ("ota_1", "app", "ota_1", 0x210000, 0x100000), + ] + + +def test_bootloader_and_table_entries_take_their_fixed_offsets(): + text = (FIXTURES / "tables" / "idf_ota_efuse_emul.csv").read_text() + partitions = read_csv(text, 0xD000, bootloader=0x2000) + by_name = {p.name: p for p in partitions} + assert (by_name["PrimaryBTLDR"].offset, by_name["PrimaryBTLDR"].size) == (0x2000, 0xB000) + assert (by_name["PrimaryPrtTable"].offset, by_name["PrimaryPrtTable"].size) == (0xD000, 0x1000) + assert by_name["nvs"].offset == 0xE000 # follows the table, not the bootloader entry + assert by_name["storage"].flags == ("encrypted",) + + +# Reading a CSV + + +def place(text, table_offset=0x8000, **offsets): + return [(p.name, p.offset, p.size) for p in read_csv(text, table_offset, **offsets)] + + +def test_blank_offsets_align_apps_to_64k_and_the_rest_to_4k(): + text = "a, data, nvs, , 0x3000\nb, app, factory, , 1M\nc, 0x40, 0x01, , 100\nd, data, , , 4K\n" + assert place(text) == [ + ("a", 0x9000, 0x3000), + ("b", 0x10000, 0x100000), + ("c", 0x110000, 100), + ("d", 0x111000, 0x1000), + ] + + +@pytest.mark.parametrize( + ("size", "value"), + [ + ("0x1000", 0x1000), + ("4096", 4096), + ("4K", 4096), + ("4k", 4096), + ("1M", 0x100000), + ("1m", 0x100000), + ("0x10K", 0x4000), + ("1_024", 1024), + ("0o10000", 4096), + ], +) +def test_numbers_read_like_the_build(size, value): + assert place(f"a, data, nvs, 0x9000, {size}\n") == [("a", 0x9000, value)] + + +def test_negative_size_gives_the_end(): + assert place("a, data, nvs, 0x9000, -0x10000\n") == [("a", 0x9000, 0x7000)] + + +def test_comments_blank_lines_byte_order_mark_and_line_ends(): + data = b"\xef\xbb\xbf# Name, Type\r\n\r\n # indented comment\r\nnvs,data,nvs,0x9000,0x6000,\r\n" + assert place(esp_idf.decode(data)) == [("nvs", 0x9000, 0x6000)] + for encoding in ("utf-16", "utf-32", "utf-8-sig"): + decoded = esp_idf.decode("nvs, data, nvs, , 0x6000\n".encode(encoding)) + assert place(decoded) == [("nvs", 0x9000, 0x6000)] + + +def test_names_flags_and_subtypes(): + text = ( + "keys, data, nvs_keys, , 4K, encrypted:readonly\n" + "Ota, APP, OTA_3, , 64K\n" + "custom, data, mine, , 4K\n" + "raw, 0x40, 0x07, , 4K\n" + "blank, data, , , 4K\n" + "extra, data, fat, , 4K, , ignored\n" + ) + partitions = read_csv(text, 0x8000) + assert [(p.type_name, p.subtype_name, p.flags) for p in partitions] == [ + ("data", "nvs_keys", ("encrypted", "readonly")), + ("app", "ota_3", ()), + ("data", "mine", ()), # registered by a component: kept as written + ("0x40", "0x07", ()), + ("data", "undefined", ()), + ("data", "fat", ()), + ] + assert partitions[2].subtype is None + + +@pytest.mark.parametrize( + ("text", "message", "hint"), + [ + ("nvs, , nvs, , 4K\n", "line 1: the type is blank", "Name, Type"), + ("nvs, code, nvs, , 4K\n", "unknown type 'code'", "app, data"), + ("Name, Type, SubType, Offset, Size\n", "unknown type 'Type'", "header line with #"), + ("a, 0x100, 0, , 4K\n", "not between 0 and 0xFF", None), + ("a, data, 0x100, , 4K\n", "not between 0 and 0xFF", "Name, Type"), + ("app, app, , , 64K\n", "an app partition needs a subtype", None), + ("a, data, nvs, 0x9000, \n", "line 1: the size is blank", None), + ("a, data, nvs, 9000h, 4K\n", "bad offset '9000h'", None), + ("a, data, nvs, , 4 KB\n", "bad size '4 KB'", None), + ("a, data, nvs, , 4K, secret\n", "unknown flag 'secret'", "encrypted and readonly"), + ("a, data, nvs, , $SIZE\n", "$SIZE is an environment variable", "replace it"), + ("a, data, nvs, , ${SIZE}\n", "${SIZE} is an environment variable", None), + ("a, data, nvs, 0x8000, 4K\n", "starts at 0x8000, but the partition table ends at 0x9000", "0x9000 or higher"), + ( + "a, data, nvs, 0x9000, 8K\nb, data, nvs, 0xa000, 4K\n", + "b starts at 0xa000, but a, listed before it, runs to 0xafff", + "make a smaller", + ), + ( + "a, data, nvs, 0x10000, 4K\nb, data, nvs, 0x9000, 4K\n", + "it is listed after a, which starts higher", + "address order", + ), + ("a, data, nvs, 0x9000, 0\n", "has size 0", None), + ("b, bootloader, primary, , \n", "bootloader offset, which is not known", "--sdkconfig"), + ("b, bootloader, recovery, , \n", "recovery bootloader offset", "RECOVERY_OFFSET"), + ], +) +def test_bad_csv(text, message, hint): + with pytest.raises(ImportProblem) as caught: + read_csv(text, 0x8000, bootloader=0x1000 if "recovery" in text else None) + assert message in str(caught.value) + if hint: + assert hint in caught.value.hint + + +def test_an_escaped_dollar_and_a_comment_with_a_variable_are_kept(): + assert place("# see $IDF_PATH\na, data, nvs, , 4K\n") == [("a", 0x9000, 4096)] + assert place("a\\$b, data, nvs, , 4K\n") == [("a\\$b", 0x9000, 4096)] + + +def test_text_that_is_not_utf8(): + with pytest.raises(ImportProblem, match="neither UTF-8"): + esp_idf.decode(b"nvs, data, \xff\xfe\xfa") + + +# Reading a binary table + + +def test_binary_round_trip_of_names_types_and_flags(): + data = table_bytes( + ("nvs", 1, 2, 0x9000, 0x6000, 0), + ("sixteen_chars_ab", 0, 0x10, 0x10000, 0x100000, 0b11), + ("raw", 0x40, 0x99, 0x110000, 0x1000, 0b100), # an unknown flag bit is ignored + ) + assert esp_idf.is_binary(data) + partitions = read_binary(data) + assert [(p.name, p.type_name, p.subtype_name, p.flags) for p in partitions] == [ + ("nvs", "data", "nvs", ()), + ("sixteen_chars_ab", "app", "ota_0", ("encrypted", "readonly")), + ("raw", "0x40", "0x99", ()), + ] + + +def test_binary_without_checksum_is_read(): + row = struct.pack("<2sBBLL16sL", b"\xaa\x50", 1, 2, 0x9000, 0x6000, b"nvs", 0) + assert [p.name for p in read_binary(row + b"\xff" * 32)] == ["nvs"] + + +@pytest.mark.parametrize( + ("change", "message"), + [ + (lambda data: data[:0x30] + b"x" + data[0x31:], "MD5 checksum does not match"), + (lambda data: data[:0x40], "no end marker"), + (lambda data: data[:0x50], "ends inside an entry"), + (lambda data: data[:0x20] + b"\x12\x34" + data[0x22:], "entry 2 of the binary table"), + ], +) +def test_bad_binary(change, message): + data = table_bytes(("nvs", 1, 2, 0x9000, 0x6000, 0), ("app", 0, 0, 0x10000, 0x10000, 0)) + with pytest.raises(ImportProblem, match=message): + read_binary(change(data)) + + +# The sdkconfig + + +def test_sdkconfig_values(): + config = read_sdkconfig( + "# comment\n" + 'CONFIG_IDF_TARGET="esp32s3"\n' + 'CONFIG_ESPTOOLPY_FLASHSIZE="16MB"\n' + "CONFIG_PARTITION_TABLE_OFFSET=0x10000\n" + "# CONFIG_BOOTLOADER_OFFSET_IN_FLASH is not set\n" + "CONFIG_BOOTLOADER_RECOVERY_OFFSET=0x3f0000\n" + ) + assert (config.flash_size, config.table_offset, config.recovery_offset) == ( + 16 * 1024 * 1024, + 0x10000, + 0x3F0000, + ) + assert (config.bootloader_offset, config.bootloader_from) == (0, "CONFIG_IDF_TARGET esp32s3") + + +@pytest.mark.parametrize( + ("lines", "offset"), + [ + ('CONFIG_IDF_TARGET="esp32"', 0x1000), + ('CONFIG_IDF_TARGET="esp32s2"', 0x1000), + ('CONFIG_IDF_TARGET="esp32c3"', 0), + ('CONFIG_IDF_TARGET="esp32p4"', 0x2000), + ('CONFIG_IDF_TARGET="esp32"\nCONFIG_BOOTLOADER_OFFSET_IN_FLASH=0x0', 0), + ("", None), + ], +) +def test_sdkconfig_bootloader_offset(lines, offset): + assert read_sdkconfig(lines).bootloader_offset == offset + + +@pytest.mark.parametrize( + ("value", "size"), + [('"4MB"', FOUR_MB), ('"512KB"', 512 * 1024), ('"detect"', None), ("", None)], +) +def test_sdkconfig_flash_size(value, size): + line = f"CONFIG_ESPTOOLPY_FLASHSIZE={value}\n" if value else "" + assert read_sdkconfig(line).flash_size == size + + +def test_sdkconfig_secure_boot_v1(): + v1 = "CONFIG_SECURE_BOOT=y\nCONFIG_SECURE_BOOT_V1_ENABLED=y\n" + assert read_sdkconfig(v1).secure_boot_v1 + assert not read_sdkconfig(v1 + "CONFIG_SECURE_BOOT_ALLOW_SHORT_APP_PARTITION=y\n").secure_boot_v1 + assert not read_sdkconfig("CONFIG_SECURE_BOOT=y\nCONFIG_SECURE_BOOT_V2_ENABLED=y\n").secure_boot_v1 + + +# Importing + + +def import_text(text, flash=FOUR_MB, table=None, sdkconfig="", **kwargs): + config = read_sdkconfig(sdkconfig) if sdkconfig else None + return import_table(text.encode(), "partitions.csv", settings(flash, table, config), **kwargs) + + +def children(imported, tmp_path): + """(id, name, start, size, kind) of each block in the flash region, in address order.""" + file = tmp_path / "layout.yaml" + file.write_text(imported.text) + result = run(file) + assert result.ok, codes(result) + flash = result.tree.index["flash"] + return [(n.id, n.name, n.start, n.size, n.kind) for n in flash.children] + + +def test_layout_without_sdkconfig(tmp_path): + imported = import_text("nvs, data, nvs, , 0x6000\nfactory, app, factory, , 1M\n") + assert children(imported, tmp_path) == [ + ("bootloader", "Bootloader", 0, 0x8000, "normal"), + ("partition_table", "Partition table", 0x8000, 0x1000, "normal"), + ("nvs", "nvs", 0x9000, 0x6000, "normal"), + ("free", "Free\n4 KB", 0xF000, 0x1000, "reserved"), + ("factory", "factory", 0x10000, 0x100000, "normal"), + ("free_2", "Free\n3008 KB", 0x110000, 0x2F0000, "reserved"), + ] + assert "from 0x0, 0x1000 or 0x2000, by chip" in imported.text + assert "Bootloader offset not known" in imported.text + assert "Flash size 4 MB, from --flash-size." in imported.text + assert "Partition table at 0x8000, from the default." in imported.text + + +def test_layout_of_an_esp32_reserves_the_space_below_the_bootloader(tmp_path): + imported = import_text("nvs, data, nvs, , 0x6000\n", sdkconfig='CONFIG_IDF_TARGET="esp32"\n') + found = children(imported, tmp_path) + assert found[:2] == [ + ("reserved", "Reserved", 0, 0x1000, "reserved"), + ("bootloader", "Bootloader", 0x1000, 0x7000, "normal"), + ] + assert "Bootloader at 0x1000, from CONFIG_IDF_TARGET esp32 in sdkconfig." in imported.text + + +def test_partition_named_like_a_block_gets_its_own_id(tmp_path): + imported = import_text("flash, data, nvs, , 0x6000\nbootloader, data, fat, , 4K\nfree, data, fat, , 4K\n") + ids = [block[0] for block in children(imported, tmp_path)] + assert len(ids) == len(set(ids)) + assert {"flash_2", "bootloader_2", "free_2"} <= set(ids) + + +def test_full_flash_has_no_free_block(tmp_path): + imported = import_text("a, data, fat, , -0x400000\n") # a negative size gives the end + ids = [block[0] for block in children(imported, tmp_path)] + assert ids == ["bootloader", "partition_table", "a"] + assert '"Flash\\n4 MB of 4 MB (100%)"' in imported.text + + +def test_table_offset_option_and_sdkconfig(tmp_path): + text = "nvs, data, nvs, , 0x6000\n" + sdkconfig = 'CONFIG_ESPTOOLPY_FLASHSIZE="2MB"\nCONFIG_PARTITION_TABLE_OFFSET=0xF000\n' + imported = import_text(text, flash=None, sdkconfig=sdkconfig) + assert "nvs\", start: 0x00010000" in imported.text + assert "Flash size 2 MB, from CONFIG_ESPTOOLPY_FLASHSIZE in sdkconfig." in imported.text + imported = import_text(text, flash=FOUR_MB, table=0x6000, sdkconfig=sdkconfig) + assert "nvs\", start: 0x00007000" in imported.text # the options win + assert "Partition table at 0x6000, from --table-offset." in imported.text + + +@pytest.mark.parametrize( + ("text", "flash", "table", "sdkconfig", "message", "hint"), + [ + ("a, data, nvs, , 4K\n", None, None, "", "the flash size is not known", "--sdkconfig"), + ("a, data, nvs, , 4K\n", None, None, 'CONFIG_IDF_TARGET="esp32"\n', "does not set CONFIG_ESPTOOLPY_FLASHSIZE", "--flash-size"), + ("a, data, nvs, , 4K\n", None, None, 'CONFIG_ESPTOOLPY_FLASHSIZE="detect"\n', "'detect', which is not a size", "--flash-size"), + ("a, app, factory, , 4M\n", FOUR_MB, None, "", "past the end of the 4 MB flash", "flash size"), + ("a, data, nvs, , 4K\n", FOUR_MB, 0x8800, "", "0x8800 from --table-offset is not a multiple", None), + ("a, data, nvs, , 4K\n", FOUR_MB, 0x8000, "CONFIG_BOOTLOADER_OFFSET_IN_FLASH=0x9000\n", "not below the partition table", None), + ], +) +def test_import_problems(text, flash, table, sdkconfig, message, hint): + with pytest.raises(ImportProblem) as caught: + import_text(text, flash, table, sdkconfig) + assert message in str(caught.value) + if hint: + assert hint in caught.value.hint + + +def test_missing_flash_size_suggests_the_sdkconfig_nearby(): + with pytest.raises(ImportProblem) as caught: + import_text("a, data, nvs, , 4K\n", None, near_sdkconfig="project/sdkconfig") + assert caught.value.hint.startswith("pass --sdkconfig project/sdkconfig") + + +@pytest.mark.parametrize( + ("rows", "message"), + [ + ([("a", 1, 2, 0x8000, 0x1000, 0)], "starts at 0x8000, below the end of the partition table"), + ([("a", 1, 2, 0x9000, 0x2000, 0), ("b", 1, 2, 0xA000, 0x1000, 0)], "b (0xa000 - 0xafff) overlaps a"), + ], +) +def test_binary_that_cannot_be_drawn(rows, message): + with pytest.raises(ImportProblem, match=message.replace("(", r"\(").replace(")", r"\)")): + import_table(table_bytes(*rows), "t.bin", settings(FOUR_MB, None)) + + +def test_binary_gives_its_own_table_and_bootloader_offsets(): + data = table_bytes( + ("boot", 2, 0, 0x2000, 0xE000, 0), + ("table", 3, 0, 0x10000, 0x1000, 0), + ("nvs", 1, 2, 0x11000, 0x6000, 0), + ) + imported = import_table(data, "t.bin", settings(FOUR_MB, None)) + assert "Partition table at 0x10000, from its own entry in the table." in imported.text + assert "Bootloader at 0x2000, from its own entry in the table." in imported.text + assert '"Reserved", kind: reserved, start: 0x00000000, size: 8K' in imported.text + assert '"Bootloader"' not in imported.text and '"Partition table"' not in imported.text + + +@pytest.mark.parametrize( + ("text", "problem", "hint"), + [ + ("a, data, nvs, 0x9800, 0x3000\n", "starts at 0x9800, not on a 4 KB boundary", "0xa000"), + ("a, app, factory, 0x18000, 64K\n", "not on a 64 KB boundary", "0x20000"), + ("a, app, factory, , 0x10800\n", "has size 0x10800, not a multiple of 4 KB", "0x11000"), + ("n, data, nvs, , 8K\n", "read-write NVS partition of 8 KB", "readonly"), + ("o, data, ota, , 8K, readonly\n", "is ota data marked readonly", "remove"), + ("c, data, coredump, , 64K, readonly\n", "is coredump data marked readonly", "remove"), + ("o, data, ota, , 4K\n", "otadata (data, ota) partition of 0x1000 bytes", "0x2000"), + ("o, data, ota, , 8K\np, data, ota, , 8K\n", "one of 2 otadata", "only one"), + ("t, data, tee_ota, , 4K\n", "TEE otadata", "0x2000"), + ("a, data, fat, , 4K\na, data, fat, , 4K\n", "shares its name with 1 more", "own name"), + ], +) +def test_esp_idf_rules(text, problem, hint): + diagnostics = import_text(text).diagnostics + assert diagnostics, text + assert diagnostics[0].code == "W113" + assert problem in diagnostics[0].message and "which current ESP-IDF rejects" in diagnostics[0].message + assert hint in diagnostics[0].hint + assert diagnostics[0].subject + + +def test_rules_that_hold(): + text = "n, data, nvs, , 8K, readonly\no, data, ota, , 8K\na, app, factory, , 0x10800\n" + assert [d.message for d in import_text(text).diagnostics] == [ + "line 3 (a) has size 0x10800, not a multiple of 4 KB, which current ESP-IDF rejects" + ] + + +def test_secure_boot_v1_needs_app_sizes_in_64k(): + sdkconfig = "CONFIG_SECURE_BOOT=y\nCONFIG_SECURE_BOOT_V1_ENABLED=y\n" + diagnostics = import_text("a, app, factory, , 68K\n", sdkconfig=sdkconfig).diagnostics + assert "not a multiple of 64 KB" in diagnostics[0].message + + +def test_binary_skips_the_rules(): + data = table_bytes(("n", 1, 2, 0x9000, 0x2000, 0)) # an 8 KB read-write NVS + assert import_table(data, "t.bin", settings(FOUR_MB, None)).diagnostics == [] + + +# The command + + +def write_project(tmp_path, sdkconfig='CONFIG_ESPTOOLPY_FLASHSIZE="4MB"\nCONFIG_IDF_TARGET="esp32c3"\n'): + table = tmp_path / "partitions.csv" + table.write_text("nvs, data, nvs, , 0x6000\nfactory, app, factory, , 1M\n") + if sdkconfig is not None: + (tmp_path / "sdkconfig").write_text(sdkconfig) + return table + + +def test_command_writes_next_to_the_table(tmp_path): + table = write_project(tmp_path) + result = invoke("from-partitions", table, "--sdkconfig", tmp_path / "sdkconfig") + assert result.exit_code == 0, result.output + assert result.stdout.strip() == str(tmp_path / "partitions.yaml") + text = (tmp_path / "partitions.yaml").read_text() + assert "Bootloader at 0x0, from CONFIG_IDF_TARGET esp32c3 in sdkconfig." in text + assert invoke("check", tmp_path / "partitions.yaml", "--strict").exit_code == 0 + again = invoke("from-partitions", table, "--sdkconfig", tmp_path / "sdkconfig") + assert again.exit_code == 2 and "use --force" in again.stderr + assert invoke("from-partitions", table, "--flash-size", "4M", "--force").exit_code == 0 + + +def test_command_json(tmp_path): + table = write_project(tmp_path) + table.write_text("n, data, nvs, , 8K\n") + out = tmp_path / "layout.yaml" + result = invoke("from-partitions", table, "-o", out, "--flash-size", "2M", "--json", "--strict") + assert result.exit_code == 1 + data = json.loads(result.stdout) + assert data["ok"] is False and data["output"] == str(out) + assert [d["code"] for d in data["diagnostics"]] == ["W113"] + assert data["diagnostics"][0]["subject"] == "n" + + +def test_command_hints_at_the_sdkconfig_next_to_the_table(tmp_path): + table = write_project(tmp_path) + result = invoke("from-partitions", table, "--json") + assert result.exit_code == 2 + diagnostic = json.loads(result.stdout)["diagnostics"][0] + assert diagnostic["code"] == "E090" + assert "the flash size is not known" in diagnostic["message"] + assert diagnostic["hint"].startswith(f"pass --sdkconfig {tmp_path / 'sdkconfig'}") + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--flash-size", "four"], "bad --flash-size"), + (["--table-offset", "-1"], "bad --table-offset"), + (["--sdkconfig", "missing"], "cannot read missing"), + (["-o", "partitions.csv"], "would overwrite the partition table"), + ], +) +def test_command_usage_problems(tmp_path, monkeypatch, args, message): + monkeypatch.chdir(tmp_path) + write_project(tmp_path) + result = invoke("from-partitions", "partitions.csv", *args) + assert result.exit_code == 2 + assert message in result.stderr + + +def test_command_on_a_missing_table(tmp_path): + result = invoke("from-partitions", tmp_path / "nope.csv", "--flash-size", "4M") + assert result.exit_code == 2 and "cannot read" in result.stderr + + +def test_command_on_a_binary_table(tmp_path): + table = tmp_path / "partition-table.bin" + table.write_bytes(table_bytes(("nvs", 1, 2, 0x9000, 0x6000, 0))) + result = invoke("from-partitions", table, "--flash-size", "4M", "--strict") + assert result.exit_code == 0, result.output + text = (tmp_path / "partition-table.yaml").read_text() + assert "an ESP-IDF binary partition table" in text + + +# Found in review + + +def test_binary_entries_agree_with_the_sdkconfig(): + data = table_bytes(("table", 3, 0, 0x10000, 0x1000, 0), ("nvs", 1, 2, 0x11000, 0x6000, 0)) + agrees = read_sdkconfig("CONFIG_PARTITION_TABLE_OFFSET=0x10000\n") + imported = import_table(data, "t.bin", settings(FOUR_MB, None, agrees)) + assert "Partition table at 0x10000, from CONFIG_PARTITION_TABLE_OFFSET in sdkconfig." in imported.text + disagrees = read_sdkconfig("CONFIG_PARTITION_TABLE_OFFSET=0x8000\n") + with pytest.raises(ImportProblem) as caught: + import_table(data, "t.bin", settings(FOUR_MB, None, disagrees)) + assert "own partition table entry is at 0x10000, but CONFIG_PARTITION_TABLE_OFFSET" in str(caught.value) + assert "belongs to the build" in caught.value.hint + + +def test_binary_bootloader_entry_against_the_option(): + data = table_bytes(("boot", 2, 0, 0x1000, 0x7000, 0), ("nvs", 1, 2, 0x9000, 0x6000, 0)) + with pytest.raises(ImportProblem, match="own bootloader entry is at 0x1000, but --bootloader-offset says 0x0"): + import_table(data, "t.bin", settings(FOUR_MB, None, bootloader_offset=0)) + + +def test_unknown_subtype_name_is_a_warning_and_a_bad_number_an_error(): + diagnostics = import_text("nvs, data, nvss, , 24K\n").diagnostics + assert [d.code for d in diagnostics] == ["W113"] + assert diagnostics[0].message == ( + "line 1 (nvs) has subtype 'nvss', which current ESP-IDF rejects unless a component " + "registers it" + ) + with pytest.raises(ImportProblem, match="bad subtype '0xZZ'"): + read_csv("a, data, 0xZZ, , 4K\n", 0x8000) + + +@pytest.mark.parametrize( + ("text", "flash", "table", "message"), + [ + ("# empty\n", 8 * 1024, None, "partition table at 0x8000 runs past the end of the 8 KB flash"), + ("", FOUR_MB, 0x400000, "runs past the end of the 4 MB flash"), + ("a, data, nvs, , 24K\n", FOUR_MB, 0, "leaves no room for the bootloader"), + ], +) +def test_generated_blocks_must_fit(text, flash, table, message): + with pytest.raises(ImportProblem, match=message): + import_text(text, flash, table) + + +def test_binary_entry_of_size_zero(): + with pytest.raises(ImportProblem, match="has size 0") as caught: + import_table(table_bytes(("z", 1, 2, 0x9000, 0, 0)), "t.bin", settings(FOUR_MB, None)) + assert "give it a size" in caught.value.hint + + +def test_negative_offset_in_a_message(): + with pytest.raises(ImportProblem, match="starts at -0x1000"): + read_csv("a, data, nvs, -0x1000, 4K\n", 0x8000) + + +def test_recovery_bootloader_slot(): + text = "nvs, data, nvs, , 24K\nrecovery, bootloader, recovery, , \n" + partitions = read_csv(text, 0x8000, bootloader=0x1000, recovery=0x3F0000) + assert (partitions[1].offset, partitions[1].size) == (0x3F0000, 0x7000) + + +def test_bootloader_option_wins_over_the_sdkconfig(): + imported = import_text("nvs, data, nvs, , 24K\n", sdkconfig='CONFIG_IDF_TARGET="esp32"\n') + assert "Bootloader at 0x1000" in imported.text + config = settings(FOUR_MB, None, read_sdkconfig('CONFIG_IDF_TARGET="esp32"\n'), bootloader_offset=0x2000) + imported = import_table(b"nvs, data, nvs, , 24K\n", "t.csv", config) + assert "Bootloader at 0x2000, from --bootloader-offset." in imported.text + + +def test_sdkconfig_with_windows_line_ends(): + config = read_sdkconfig('CONFIG_ESPTOOLPY_FLASHSIZE="8MB"\r\nCONFIG_PARTITION_TABLE_OFFSET=0x9000\r\n') + assert (config.flash_size, config.table_offset) == (8 * 1024 * 1024, 0x9000) + + +@pytest.mark.parametrize("size", ["4M", "4MB", "4m", "0x400000", "4194304"]) +def test_command_flash_size_spellings(tmp_path, size): + table = write_project(tmp_path, sdkconfig=None) + result = invoke("from-partitions", table, "--flash-size", size, "--force") + assert result.exit_code == 0, result.output + assert "Flash size 4 MB, from --flash-size." in (tmp_path / "partitions.yaml").read_text() + + +def test_command_with_a_bootloader_entry_and_the_option(tmp_path): + table = tmp_path / "partitions.csv" + table.write_text("boot, bootloader, primary, , \nnvs, data, nvs, , 24K\n") + result = invoke("from-partitions", table, "--flash-size", "4M") + assert result.exit_code == 2 and "--bootloader-offset" in result.stderr + result = invoke("from-partitions", table, "--flash-size", "4M", "--bootloader-offset", "0x1000") + assert result.exit_code == 0, result.output + text = (tmp_path / "partitions.yaml").read_text() + assert '"boot", start: 0x00001000, size: 28K' in text + + +@pytest.mark.parametrize("path", [".", "folder"]) +def test_command_on_a_path_that_is_not_a_file(tmp_path, monkeypatch, path): + monkeypatch.chdir(tmp_path) + (tmp_path / "folder").mkdir() + result = invoke("from-partitions", path, "--flash-size", "4M", "--json") + assert result.exit_code == 2 + assert json.loads(result.stdout)["diagnostics"][0]["code"] == "E090" + + +def test_command_reports_a_crash_as_exit_3(tmp_path, monkeypatch): + def crash(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr("memmap.cli.import_table", crash) + table = write_project(tmp_path, sdkconfig=None) + result = invoke("from-partitions", table, "--flash-size", "4M") + assert result.exit_code == 3 and "internal error: boom" in result.stderr