From 6db8b27df01fed4972bd4d08a2ed9cd7ec34b49e Mon Sep 17 00:00:00 2001 From: iharhl Date: Sat, 14 Nov 2026 11:20:10 +0100 Subject: [PATCH] Read Zephyr and nRF Connect SDK partitions, add region spaces `memmap from-partitions` now also reads Zephyr's merged zephyr.dts and nRF Connect SDK Partition Manager files (partitions.yml, pm_static.yml), telling the formats apart by content. - Devicetree: one column per flash with fixed-partitions, addresses mapped through `ranges`, erase-block-size as sectors, fixed-subpartitions nested and zoomed into, and notes for MCUboot's boot partition, slots and scratch. SPI, QSPI and FlexSPI flash is drawn in its own space from 0. - Partition Manager: one column per region, `span` containers nested, and notes for MCUboot, the image header, NSIB, TF-M and the app. - W114: a partition that partly overlaps another or lies outside its flash is left out. A stale `app` in pm_static.yml gives way. - Regions take an optional top-level `space:`. Regions in different spaces may overlap, placement and gap filling run per space, and a `/` column draws only the default space. Checked against the generated headers of real builds: 1395 devicetree builds (7521 entries) and 222 Partition Manager builds (2144 entries), all matching. Five fixtures cover what the readers need; docs/from-partitions.md has the details. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 30 +- README.md | 40 +- docs/from-partitions.md | 186 +- memmap/cli.py | 31 +- memmap/describe.py | 4 + memmap/from_partitions/__init__.py | 8 +- memmap/from_partitions/areas.py | 252 ++ memmap/from_partitions/dts.py | 402 +++ memmap/from_partitions/importer.py | 105 +- memmap/from_partitions/partition_manager.py | 202 ++ memmap/from_partitions/writer.py | 130 +- memmap/from_partitions/zephyr.py | 259 ++ memmap/resolve.py | 72 +- memmap/schema.py | 10 + memmap/validate.py | 2 +- memmap/view.py | 34 +- .../mcxn947_flexspi/devicetree_generated.h | 12 + .../builds/mcxn947_flexspi/zephyr.dts | 2418 +++++++++++++++++ .../devicetree_generated.h | 25 + .../builds/nrf5340_subpartitions/zephyr.dts | 1262 +++++++++ .../builds/nrf5340_tfm_pm/partitions.yml | 155 ++ .../builds/nrf5340_tfm_pm/pm_config.h | 92 + .../devicetree_generated.h | 14 + .../builds/stm32_mcuboot_scratch/zephyr.dts | 1050 +++++++ tests/fixtures/zephyr_partitions/index.json | 68 + tests/fixtures/zephyr_partitions/refresh.py | 175 ++ .../tables/pm_static_stale_app.yml | 101 + tests/golden/partitions/mcxn947_flexspi.svg | 30 + tests/golden/partitions/mcxn947_flexspi.yaml | 41 + .../partitions/nrf5340_subpartitions.yaml | 54 + tests/golden/partitions/nrf5340_tfm_pm.svg | 133 + tests/golden/partitions/nrf5340_tfm_pm.yaml | 104 + .../partitions/pm_static_stale_app.yaml | 79 + .../partitions/stm32_mcuboot_scratch.yaml | 44 + tests/test_cli.py | 1 + tests/test_from_zephyr.py | 761 ++++++ tests/test_resolve.py | 94 + 37 files changed, 8359 insertions(+), 121 deletions(-) create mode 100644 memmap/from_partitions/areas.py create mode 100644 memmap/from_partitions/dts.py create mode 100644 memmap/from_partitions/partition_manager.py create mode 100644 memmap/from_partitions/zephyr.py create mode 100644 tests/fixtures/zephyr_partitions/builds/mcxn947_flexspi/devicetree_generated.h create mode 100644 tests/fixtures/zephyr_partitions/builds/mcxn947_flexspi/zephyr.dts create mode 100644 tests/fixtures/zephyr_partitions/builds/nrf5340_subpartitions/devicetree_generated.h create mode 100644 tests/fixtures/zephyr_partitions/builds/nrf5340_subpartitions/zephyr.dts create mode 100644 tests/fixtures/zephyr_partitions/builds/nrf5340_tfm_pm/partitions.yml create mode 100644 tests/fixtures/zephyr_partitions/builds/nrf5340_tfm_pm/pm_config.h create mode 100644 tests/fixtures/zephyr_partitions/builds/stm32_mcuboot_scratch/devicetree_generated.h create mode 100644 tests/fixtures/zephyr_partitions/builds/stm32_mcuboot_scratch/zephyr.dts create mode 100644 tests/fixtures/zephyr_partitions/index.json create mode 100644 tests/fixtures/zephyr_partitions/refresh.py create mode 100644 tests/fixtures/zephyr_partitions/tables/pm_static_stale_app.yml create mode 100644 tests/golden/partitions/mcxn947_flexspi.svg create mode 100644 tests/golden/partitions/mcxn947_flexspi.yaml create mode 100644 tests/golden/partitions/nrf5340_subpartitions.yaml create mode 100644 tests/golden/partitions/nrf5340_tfm_pm.svg create mode 100644 tests/golden/partitions/nrf5340_tfm_pm.yaml create mode 100644 tests/golden/partitions/pm_static_stale_app.yaml create mode 100644 tests/golden/partitions/stm32_mcuboot_scratch.yaml create mode 100644 tests/test_from_zephyr.py diff --git a/AGENTS.md b/AGENTS.md index d63716e..89623bd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,11 +69,32 @@ positions in the flash chip, drawn from 0, not CPU addresses. The column also ho bootloader, the partition table and `Free` blocks; the header comments say where each setting came from. https://github.com/iharhl/memmap/blob/main/docs/from-partitions.md has the details. +## Typical task: map a Zephyr or nRF Connect SDK flash layout +1. Find the build's layout. Zephyr: `build/zephyr/zephyr.dts`, the merged devicetree + (`build//zephyr/zephyr.dts` under sysbuild), not a board `.dts` or overlay. nRF Connect + SDK with the Partition Manager: `build/partitions.yml`, or the project's `pm_static.yml`. +2. `memmap from-partitions build/zephyr/zephyr.dts -o layout.yaml --json --strict`. It needs no + options, and the ESP-IDF ones are refused. +3. Fix W114 (a partition left out: it partly overlaps another or lies outside its flash) by + correcting the partition in the overlay or pm_static.yml. W100 marks a partition edge + inside an erase block. +4. Continue from step 4 of the first task. + +Each flash with `fixed-partitions`, or each Partition Manager region, is a region and column. +A flash on a SPI or QSPI bus, and Partition Manager's `external_flash`, has no CPU address: it +gets a `space` of its own and is drawn from 0. A partition inside another, a Zephyr +`fixed-subpartitions` node or a Partition Manager container (`span`), is drawn inside it and +zoomed into. Notes give MCUboot's slots, scratch and image header. +https://github.com/iharhl/memmap/blob/main/docs/from-partitions.md has the details. + ## Rules - Never guess keys; run `memmap schema` when unsure. - Ranges are inclusive: end = start + size - 1. - Prefer `size` over `end`, and sequential placement over explicit `start`. - Display tweaks go in `view`, never by changing `memory`. +- One address space per drawing, except top-level regions with a `space`: memory the CPU does + not map, such as an external flash from 0. They may overlap the others, and a `/` column + leaves them out (E020 when no region is left). Without `view.columns`, each gets a column. - The look is fixed on purpose: font family, address format, outlines and label side have no keys. Connectors only choose solid or dashed lines (`style.connectors`). - Use `--strict` on `check` and `render` to fail on warnings. Without it they exit 0 with @@ -97,6 +118,7 @@ came from. https://github.com/iharhl/memmap/blob/main/docs/from-partitions.md ha - Label one edge: view.overrides: [{ region: ahb1, top_label: end }] - Block height: view.overrides: [{ region: boot, height: 48 }] - Multi-line note: note: "line one\nline two" (long notes also wrap) +- External flash at 0: space: qspi on its top-level region, then a column with root: qspi Quote `--set` values that contain `[` or `{`, since shells expand them. @@ -149,7 +171,7 @@ Every diagnostic has a stable code, a YAML path, a line and usually a `hint`. In | E013 | More than one `auto` size, or nothing left for `auto` | | E014 | Gap inside a region with `fill_gaps: error` | | E015 | Too many flash sectors | -| E020 | Column root, expansion or override target does not exist | +| E020 | Column root, expansion or override target does not exist, or a `/` column with only `space` regions | | E021 | Expansion `to` column has a different root than `region` | | E022 | Expanded region is not drawn in the `from` column | | E023 | Duplicate column id | @@ -167,6 +189,7 @@ Every diagnostic has a stable code, a YAML path, a line and usually a `hint`. In | W111 | `from-elf`: section overlaps another one, left out | | W112 | `from-elf`: map region left out (no size, past the address space, overlap, too small) | | W113 | `from-partitions`: the CSV breaks a rule of ESP-IDF's partition table check | +| W114 | `from-partitions`: a Zephyr or Partition Manager partition left out (partial overlap, outside its flash) | | E090 | Usage or IO failure, such as a missing file or bad `--set` (`--json` only) | | E091 | Render or internal failure, such as missing cairo (`--json` only) | @@ -182,7 +205,8 @@ Exit codes: `0` ok, `1` validation errors, `2` usage or IO error, `3` render err tests/test_linker.py`. docs/from-elf.md describes them and how to add an image or a map. - After an intended change to `from-partitions` or the drawing, regenerate its goldens: - `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_partitions.py`. docs/from-partitions.md - describes the real ESP-IDF builds they come from. + `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_partitions.py tests/test_from_zephyr.py`. + docs/from-partitions.md describes the real ESP-IDF, Zephyr and nRF Connect SDK builds they + come from. - A new diagnostic code or recipe goes in this file too. `memmap guide` ships it to users, and `tests/test_cli.py` covers the recipes. diff --git a/README.md b/README.md index 94b8dd8..456c480 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # memmap Draw datasheet-style memory maps of MCU memory. You describe the layout in a YAML file, or -have `memmap` write that file from a linked firmware image or an ESP-IDF partition table. Either +have `memmap` write that file from a linked firmware image or a flash partition layout. Either way you get the same config, which `memmap` checks and renders to SVG, PNG or PDF. - **Write it by hand.** One YAML file holds the memory (`memory`) and how to draw it (`view`), @@ -10,7 +10,8 @@ way you get the same config, which `memmap` checks and renders to SVG, PNG or PD Keil armlink or IAR), and places every section at its real address and size in the chip's flash and RAM regions. - **Generate it from a partition table.** `from-partitions` reads an ESP-IDF partition table, CSV - or binary, and places every partition where the ESP-IDF build would. + or binary, and places every partition where the ESP-IDF build would. It also reads a Zephyr + build's devicetree and an nRF Connect SDK Partition Manager layout, with MCUboot's slots marked. - **Check before drawing.** Validation catches overlaps, overflow and flash sector misalignment, with the YAML path, line and a hint for each. - **Verify as text.** Output is deterministic, and `describe` lists everything drawn, so agents and @@ -111,6 +112,11 @@ Children without a `start` are placed one after another, and `auto` takes what i take an optional `title`, drawn above them. For a detail column created by `view.expand`, set it on the expansion: `expand: [{ region: flash, title: Flash }]`. +A config draws one address space. Memory the CPU does not map, such as an external SPI flash +whose offsets start at 0, goes in a top-level region with a `space`, so it may overlap the +others: `{ id: qspi, space: qspi, start: 0, size: 8M }`. A `/` column leaves such regions out, +so draw them in a column with their id as `root`. A config without `view.columns` gets one. + `memmap schema` prints every key with a description and an example. Point your editor's YAML extension at it for completion. [AGENTS.md](AGENTS.md) has recipes for common tweaks. @@ -138,12 +144,16 @@ ordinary config: rename or merge blocks, add `view` options, then `render` it. [docs/from-elf.md](docs/from-elf.md) covers what is read from each map, what happens when it is not enough, and how sections become blocks. -## Generating a config from an ESP-IDF partition table +## Generating a config from a partition layout + +`from-partitions` draws a flash partition layout. It reads an ESP-IDF partition table, a Zephyr +build's merged devicetree or an nRF Connect SDK Partition Manager file, and tells them apart by +their content. -`from-partitions` draws the flash of an ESP-IDF project: the bootloader, the partition table, -each partition and the free space between them. It reads the partition table CSV the project -names in `CONFIG_PARTITION_TABLE_FILENAME`, or the `build/partition_table/partition-table.bin` -the build writes. Partitions with a blank offset are placed as the build places them, so the +For an ESP-IDF project, it draws the bootloader, the partition table, each partition and the +free space between them. It reads the partition table CSV the project names in +`CONFIG_PARTITION_TABLE_FILENAME`, or the `build/partition_table/partition-table.bin` the build +writes. Partitions with a blank offset are placed as the build places them, so the drawing matches the binary table. The table does not record the flash size, so pass the project's `sdkconfig`, which also gives @@ -154,6 +164,17 @@ memmap from-partitions partitions.csv -o layout.yaml --flash-size 4M ``` A CSV that breaks a rule of ESP-IDF's own table check gets W113, with a hint. + +For Zephyr, pass `build/zephyr/zephyr.dts`, the devicetree the build merged. Each flash with +`fixed-partitions` gets a column, with MCUboot's slots marked. An external SPI or QSPI flash is +drawn from 0, in an address space of its own. For the nRF Connect SDK, pass +`build/partitions.yml` or a `pm_static.yml`: each region gets a column, and containers such as +`mcuboot_primary` are zoomed into. Neither needs options: + +```bash +memmap from-partitions build/zephyr/zephyr.dts -o layout.yaml +``` + [docs/from-partitions.md](docs/from-partitions.md) covers what is read and drawn, and the limits. ## Commands @@ -164,7 +185,7 @@ A CSV that breaks a rule of ESP-IDF's own table check gets W113, with a hint. | `memmap check CONFIG [--json] [--strict]` | Validate only | | `memmap describe CONFIG [--json]` | Resolved regions, blocks, labels and option origins | | `memmap from-elf ELF [-o OUT] [--map FILE] [--region id=start:size[:name]] [--min-size N] [--force] [--json] [--strict]` | Write a config from a linked ELF, then check it | -| `memmap from-partitions TABLE [-o OUT] [--sdkconfig FILE] [--flash-size SIZE] [--table-offset OFFSET] [--bootloader-offset OFFSET] [--force] [--json] [--strict]` | Write a config from an ESP-IDF partition table, then check it | +| `memmap from-partitions TABLE [-o OUT] [--sdkconfig FILE] [--flash-size SIZE] [--table-offset OFFSET] [--bootloader-offset OFFSET] [--force] [--json] [--strict]` | Write a config from an ESP-IDF, Zephyr or Partition Manager partition layout, then check it | | `memmap schema` | JSON Schema of the config | | `memmap guide` | Print the agent guide, for agents using an installed copy | @@ -175,7 +196,8 @@ scripts and agents. - [AGENTS.md](AGENTS.md): task guide, recipes and the full diagnostic list. `memmap guide` prints it. - [docs/from-elf.md](docs/from-elf.md): drawing a linked firmware image, its map readers and limits. -- [docs/from-partitions.md](docs/from-partitions.md): drawing an ESP-IDF partition table. +- [docs/from-partitions.md](docs/from-partitions.md): drawing an ESP-IDF, Zephyr or Partition + Manager partition layout. - [Examples](#examples): one config per feature set, below. ## Examples diff --git a/docs/from-partitions.md b/docs/from-partitions.md index ea40878..54a4f41 100644 --- a/docs/from-partitions.md +++ b/docs/from-partitions.md @@ -1,22 +1,38 @@ -# Drawing an ESP-IDF partition table +# Drawing a flash partition layout -`memmap from-partitions` writes a config from an ESP-IDF partition table: one flash column with -the bootloader, the partition table, every partition and the free space between them. +`memmap from-partitions` writes a config from a flash partition layout. It reads three formats, +told apart by their content: + +- an ESP-IDF partition table, CSV or binary: one flash column with the bootloader, the + partition table, every partition and the free space between them; +- a Zephyr build's merged devicetree, `zephyr.dts`: a column per flash with partitions; +- an nRF Connect SDK Partition Manager file, `partitions.yml` or `pm_static.yml`: a column per + memory region, with containers zoomed into. ```bash memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig ``` +```bash +memmap from-partitions build/zephyr/zephyr.dts -o layout.yaml +``` + ![The flash of an ESP32 HomeKit project, drawn by from-partitions](../tests/golden/partitions/esp32_homekit_custom_type.svg) -- [What it reads](#what-it-reads) -- [Where the settings come from](#where-the-settings-come-from) -- [What is drawn](#what-is-drawn) -- [Checks](#checks) -- [Limits](#limits) +- [ESP-IDF](#esp-idf) + - [What it reads](#what-it-reads) + - [Where the settings come from](#where-the-settings-come-from) + - [What is drawn](#what-is-drawn) + - [Checks](#checks) + - [Limits](#limits) +- [Zephyr devicetree](#zephyr-devicetree) +- [nRF Connect SDK Partition Manager](#nrf-connect-sdk-partition-manager) +- [Nesting and overlaps](#nesting-and-overlaps) - [Testing](#testing) -## What it reads +## ESP-IDF + +### What it reads The table is either of the files ESP-IDF works from, told apart by their content: @@ -47,7 +63,7 @@ ESP-IDF's `gen_esp32part.py` does: A binary table is a list of 32-byte entries with an MD5 entry and `0xFF` padding. memmap checks the MD5 and reads the entries as they are. -## Where the settings come from +### Where the settings come from The table gives the partitions, but not how big the flash is, where the partition table sits or where the ROM loads the bootloader from. memmap takes each from the options, else from the @@ -72,7 +88,7 @@ and, for a drawing that places the bootloader, `--bootloader-offset`. When the flash size is missing and an `sdkconfig` lies next to the table, the hint names it. -## What is drawn +### What is drawn One region, `flash`, from 0 to the flash size. Offsets are positions in the flash chip, not the addresses the CPU reads the flash through. Its blocks, in address order: @@ -91,7 +107,7 @@ in place of the generated block. The region has 4K `sectors`, so `check` reports edge inside an erase sector as W100; the tick marks are turned off in `style`. The column title gives how much of the flash the bootloader, the table and the partitions take. -## Checks +### Checks memmap stops, writing nothing, when the table cannot be drawn: @@ -120,7 +136,7 @@ A CSV that can be drawn but breaks a rule of ESP-IDF's own table check gets W113 Some of these rules are newer than others: a table that ESP-IDF 5.2 built can get W113 from a rule added later. A binary table is not checked, as the build that wrote it checked it already. -## Limits +### Limits - **Environment variables.** The build expands `$NAME` and `${NAME}` in a CSV from the environment. memmap stops at any `$` followed by a name in a partition line, and asks for @@ -129,12 +145,96 @@ rule added later. A binary table is not checked, as the build that wrote it chec read it as 0, which is also `otadata`'s subtype. memmap follows ESP-IDF 5. - **Flash size `detect`.** Older sdkconfigs can leave the size to be detected when flashing. Give `--flash-size` then. -- **Other formats.** Only ESP-IDF tables are read. Arduino-ESP32, PlatformIO and MicroPython use - the same CSV format, so their tables work too. +- **Other formats.** Arduino-ESP32, PlatformIO and MicroPython use ESP-IDF's CSV format, so + their tables work too. Zephyr and Partition Manager layouts are covered below. - **Table length.** ESP-IDF's binary table holds at most 95 entries. memmap does not check this. +## Zephyr devicetree + +memmap reads the devicetree the build merged and preprocessed, `build/zephyr/zephyr.dts`, or +`build//zephyr/zephyr.dts` under sysbuild. A board `.dts`, an overlay or +`zephyr.dts.pre` needs the C preprocessor, includes and board files, so memmap stops on one and +asks for the merged file. + +Every node with `compatible = "fixed-partitions"` lists the partitions of its parent, a flash +device, and each such flash becomes a region and a column: + +| Flash | Example | Base and size | +| --- | --- | --- | +| Mapped by the CPU | `soc-nv-flash` on nRF, STM32, ESP32, RP2040 | its `reg`, mapped through the `ranges` above it, as Zephyr maps it | +| A chip on a SPI, QSPI or FlexSPI bus | `jedec,spi-nor`, `nordic,qspi-nor`, `nxp,imx-flexspi-nor` | from 0, with the size its `size` property gives in bits | + +A bus flash has no CPU address: its region gets a `space` of its own, so it may overlap the +internal flash at 0. An older STM32 QSPI board gives the size in bytes as the second cell of +`reg`; memmap reads that too. A flash with neither is drawn up to its last partition. + +- A partition's `reg` is an offset into its flash, so memmap adds the flash's base address. A + `fixed-subpartitions` partition, as the nRF5340 and nRF91 boards use for TF-M, holds + partitions whose `reg` its `ranges` maps; they are drawn inside it and zoomed into. +- Ids come from the node label (`slot0_partition`), names from the `label` property + (`image-0`), else the node label. +- Notes give MCUboot's use of each partition, by the node labels Zephyr gives them. A board + defines these partitions whether or not the build uses MCUboot, so the note says what the + partition is for, not that MCUboot is there: `boot_partition` is + MCUboot, `slotN_partition` a primary (even N) or secondary (odd N) slot of image N/2, + `slotN_ns_partition` the non-secure image, `scratch_partition` the swap scratch area. A + prefix such as `cpuapp_` is allowed. The partition that `/chosen/zephyr,code-partition` + names is marked `code partition`, and a `read-only` one `read-only`. +- `erase-block-size` gives the region's `sectors`, so `check` reports a partition edge inside + an erase block as W100. When the erase blocks do not divide the flash, or there would be more + than 4096 of them, the region gets no `sectors`, and a header comment says so. +- A flash whose node, or a node above it, has `status = "disabled"` is still drawn, as its + partitions still say how the flash is laid out, and the header says so. + +The header comments say where each flash's base and size came from. memmap does not know which +upgrade mode MCUboot uses, so it checks no slot sizes. It does not mark the image header +(`CONFIG_ROM_START_OFFSET`) or the trailer either: neither is in the devicetree. + +The ESP-IDF options do not apply, and memmap stops when one is given. + +## nRF Connect SDK Partition Manager + +An nRF Connect SDK build that uses the Partition Manager lays out flash in +`build/partitions.yml` (and `build/partitions_CPUNET.yml` for the nRF5340 network core), not +in the devicetree. A `pm_static.yml` has the same format. Each top-level key is a partition: + +- `address`, `size` and `end_address` (exclusive). A size wins over an `end_address` that + disagrees with it, and the header says so. A container that gives no address gets the range + of the partitions it spans. +- `region`, `flash_primary` when left out. Each region becomes a region and column: Flash, + External flash, SRAM, OTP, then others by name. The file does not say how big a region is, + so each is drawn from its lowest to its highest partition. +- `span` makes the entry a container of the partitions it names, such as `mcuboot_primary` + holding `mcuboot_pad` and `app`. It is drawn with them inside it and zoomed into. A + container may span containers, listed in any order. +- A negative number, a `region` that is not a name, or an `end_address` below the `address` + of an entry without a `size` stops the import. Flow style, `app: {address: 0x0, size: 0x1000}`, works too. +- `external_flash` is a chip, drawn in an address space of its own from its offsets. Any other + region that overlaps one already drawn, such as `ram_flash`, gets one too. + +Notes give the role of the partitions the nRF Connect SDK names: MCUboot and its slots, +`mcuboot_pad` (MCUboot image header), the nRF Secure Immutable Bootloader (`b0`, `s0`, `s1`), +TF-M and the application. A partition of size 0 is left out, with a header comment. + +## Nesting and overlaps + +memmap draws a tree, but both formats can list areas that overlap: + +- An area inside another becomes its child, and the outer one gets a detail column. +- Areas with the same range become one block, named after the partition, with `also ...` in + its note: Partition Manager's `tfm_nonsecure` spans exactly `app`. +- A partition that only partly overlaps another, or lies outside its flash, is left out with + W114. A container that partly overlaps another is left out with a header comment, as it only + groups partitions: with TF-M, `tfm_secure` straddles `mcuboot_primary_app`. +- In a `pm_static.yml`, `app` gives way to the partition or container it overlaps, with W114: + the build places `app` in the space the others leave, so its entry there can be stale. +- W114 names the region in `subject`, and the partition by its devicetree line or its + Partition Manager name. + ## Testing +### ESP-IDF + `tests/test_from_partitions.py` runs on real tables, kept in `tests/fixtures/esp_partitions/` with their sources and SHA-256 in `index.json`: @@ -164,3 +264,59 @@ Before the fixtures were chosen, the same comparison ran on 121 ESP-IDF builds f memmap placed 117 exactly as their `partition-table.bin`. Two differed only in a blank subtype, which their older ESP-IDF read as 0. The `.bin` of the other two came from a different table than the one committed, a build left stale. + +### Zephyr and Partition Manager + +`tests/test_from_zephyr.py` runs on real builds, kept in `tests/fixtures/zephyr_partitions/` +with their sources and SHA-256 in `index.json`: + +- **Builds.** Three Zephyr builds that committed `zephyr.dts` with the + `devicetree_generated.h` the build wrote from it, and one nRF Connect SDK build with + `partitions.yml` and `pm_config.h`. The header is the ground truth: every partition and + flash `reg` memmap reads must match its `DT_N_..._REG_IDX_0_VAL_ADDRESS` and `_SIZE` macros, + and every Partition Manager entry its `PM__ADDRESS` and `_SIZE`. The headers keep only + those lines. Between them: `fixed-subpartitions` mapped by `ranges` (nRF5340), a flash at + 0x10000000 through two levels of `ranges` with a FlexSPI NOR on a bus (NXP MCX N947), an + STM32 at 0x08000000 with MCUboot's scratch partition and a QSPI NOR sized by its `reg`, and + TF-M's nested containers with an external flash, OTP and a `ram_flash` region (nRF5340). +- **Tables.** A `pm_static.yml`, which no build generates a header from, whose stale `app` + overlaps MCUboot: W114. + +Each one is imported to a golden config in `tests/golden/partitions/`, and the MCX N947 and +nRF5340 Partition Manager builds are also drawn to a golden SVG. To add one, add it to +`refresh.py` in the fixtures folder, then: + +```bash +uv run python tests/fixtures/zephyr_partitions/refresh.py --download +``` + +```bash +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_from_zephyr.py +``` + +Before the fixtures were chosen, the same comparison ran on every build found on GitHub with +`gh search code` that committed both files: + +- 1395 distinct `zephyr.dts` and header pairs from 311 repositories, 7521 `reg` entries: all + match. In 8 of the pairs, from Zephyr 2.2, the header uses an older macro scheme that gives + the flash's address but not the partitions'. A first run missed the `ranges` mapping, which + showed as mismatches on the MCX N947 flash and on nRF5340 and nRF91 subpartitions. +- 222 distinct `partitions.yml` and `pm_config.h` pairs from 86 repositories, 2144 entries: + all match. + +A wider corpus then ran through the importer and `check`: 581 `zephyr.dts`, 98 +`partitions.yml` and 368 `pm_static.yml` files from 458 repositories, distinct by content. None crashed. Each error +and warning was checked by hand: + +- 36 devicetrees have no `fixed-partitions` node: nRF Connect SDK builds that use the + Partition Manager, and ESP32, STM32 and RP2040 boards without partitions. Three `.yml` files + were not Partition Manager files. +- W114 in 21 files, each a real problem: STM32 overlays whose MCUboot slots run past the flash + (128 KB, or 808 KB on an STM32WB55 whose top holds the radio stack), two boards whose NOR + `size` disagrees with their partitions (one an upstream Zephyr board), user overlays with + partitions that partly overlap, and `pm_static.yml` files whose `app` overlaps MCUboot or + outgrows its slot. Two of those are nRF Desktop configurations in the nRF Connect SDK itself, + whose `app` is as big as the whole slot, header included; the build shrinks it. +- W100 in 27 files, each a real edge inside an erase block: RP2040's 256-byte second-stage + bootloader, odd-sized Realtek partitions, STM32H7 partitions off its 128 KB blocks. +- W105 in 2 files: a name too long to fit the column without a break. diff --git a/memmap/cli.py b/memmap/cli.py index 8935c73..3549798 100644 --- a/memmap/cli.py +++ b/memmap/cli.py @@ -396,7 +396,8 @@ def _table_settings( @app.command( name="from-partitions", - epilog="\b\nExample:\n memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig", + epilog="\b\nExamples:\n memmap from-partitions partitions.csv -o layout.yaml --sdkconfig sdkconfig\n" + " memmap from-partitions build/zephyr/zephyr.dts -o layout.yaml", ) @click.argument("table", type=click.Path(path_type=Path)) @click.option( @@ -405,19 +406,21 @@ def _table_settings( @click.option( "--sdkconfig", type=click.Path(path_type=Path), - help="The project's sdkconfig: gives the flash size, the partition table offset and the " - "bootloader offset.", + help="ESP-IDF: the project's sdkconfig, which gives the flash size, the partition table " + "offset and the bootloader offset.", +) +@click.option( + "--flash-size", metavar="SIZE", help="ESP-IDF: flash size, such as 4M. Overrides --sdkconfig." ) -@click.option("--flash-size", metavar="SIZE", help="Flash size, such as 4M. Overrides --sdkconfig.") @click.option( "--table-offset", metavar="OFFSET", - help="Offset of the partition table. Default: from --sdkconfig, else 0x8000.", + help="ESP-IDF: offset of the partition table. Default: from --sdkconfig, else 0x8000.", ) @click.option( "--bootloader-offset", metavar="OFFSET", - help="Where the ROM loads the bootloader from: 0x1000 on the ESP32 and ESP32-S2, 0x2000 on " + help="ESP-IDF: where the ROM loads the bootloader from: 0x1000 on the ESP32 and ESP32-S2, 0x2000 on " "the ESP32-C5 and ESP32-P4, else 0. Default: from --sdkconfig.", ) @click.option("--force", is_flag=True, help="Overwrite the output if it exists.") @@ -434,11 +437,19 @@ def from_partitions( as_json: bool, strict: bool, ) -> None: - """Write a config from an ESP-IDF partition table. + """Write a config from a flash partition layout. + + TABLE is one of, told apart by its content: + + \b + - an ESP-IDF partition table: the CSV the project names in + CONFIG_PARTITION_TABLE_FILENAME, or the partition-table.bin the build writes. It does + not say how big the flash is, so give --sdkconfig or --flash-size. + - a Zephyr build's merged devicetree, build/zephyr/zephyr.dts. + - an nRF Connect SDK Partition Manager file: build/partitions.yml or a pm_static.yml. - TABLE is the partition table: the CSV the project names in CONFIG_PARTITION_TABLE_FILENAME, - or the partition-table.bin the build writes. The table does not say how big the flash is, - so give --sdkconfig or --flash-size. The written config is then checked like `memmap check`. + The options are for ESP-IDF tables only. The written config is then checked like + `memmap check`. """ try: target = output or table.with_suffix(".yaml") diff --git a/memmap/describe.py b/memmap/describe.py index 6248135..dbba892 100644 --- a/memmap/describe.py +++ b/memmap/describe.py @@ -25,6 +25,8 @@ def _region(node: Node, tree: Tree) -> dict[str, Any]: "size": node.size, "generated": node.generated, } + if node.space is not None and node.parent is not None and node.parent.parent is None: + data["space"] = node.space if node.children: data["children"] = [_region(child, tree) for child in node.children] return data @@ -113,6 +115,8 @@ def describe_data(result: Result) -> dict[str, Any]: def _tree_lines(node: Node, tree: Tree, depth: int = 0) -> list[str]: """One text line per region, indented by depth.""" tag = " [generated]" if node.generated else "" + if depth == 1 and node.space is not None: + tag += f" [space {node.space}]" kind = "" if node.kind == "normal" else f" {node.kind}" name = node.name.replace("\n", " ") lines = [ diff --git a/memmap/from_partitions/__init__.py b/memmap/from_partitions/__init__.py index 30486b0..1ac4f42 100644 --- a/memmap/from_partitions/__init__.py +++ b/memmap/from_partitions/__init__.py @@ -1,8 +1,12 @@ -"""`memmap from-partitions`: write a config from a flash partition table. +"""`memmap from-partitions`: write a config from a flash partition layout. - `esp_idf` reads an ESP-IDF partition table, CSV or binary, and places its partitions. - `sdkconfig` reads the flash size and offsets from an ESP-IDF project's sdkconfig. -- `importer` lays the partitions out in the flash, and `writer` writes the YAML. +- `dts` parses a devicetree source file, and `zephyr` reads the flash partitions of a Zephyr + build's merged zephyr.dts. +- `partition_manager` reads an nRF Connect SDK Partition Manager file. +- `areas` nests Zephyr and Partition Manager partitions into one region per flash or region. +- `importer` tells the formats apart and lays the partitions out, and `writer` writes the YAML. """ from .importer import Settings, import_table, settings diff --git a/memmap/from_partitions/areas.py b/memmap/from_partitions/areas.py new file mode 100644 index 0000000..62b141e --- /dev/null +++ b/memmap/from_partitions/areas.py @@ -0,0 +1,252 @@ +"""Lay out Zephyr and Partition Manager partitions: one region per flash device or memory +region, partitions nested where one holds another, free space between them. + +Both formats can list areas that overlap. A Partition Manager container (an entry with `span`) +holds the partitions it spans, and a devicetree may give two partitions the same range or put +one inside another. memmap draws a tree, so an area inside another becomes its child, areas with +the same range become one block, and an area that only partly overlaps another is left out: +with W114 for a partition, with a header comment for a container, which only groups others. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..diagnostics import Diagnostic +from ..from_elf.blocks import Bucket, Item +from ..from_elf.regions import ImportProblem, MemorySpec, unique_id +from ..from_elf.sizes import approx_size, floor_size + +MAX_SECTORS = 4096 # more erase blocks than this are not worth a sectors entry + + +@dataclass +class Area: + """A partition, or a Partition Manager container that groups partitions. `key` is the id to + derive the block's id from, `where` names it in messages, and `start` is absolute. A `weak` + partition is one the build places in the space the others leave, such as Partition + Manager's `app`: it gives way when it partly overlaps another. `hint` then says why.""" + + key: str + name: str + start: int + size: int + where: str + note: str | None = None + container: bool = False + weak: str | None = None + aliases: list[str] = field(default_factory=list) + children: list[Area] = field(default_factory=list) + + @property + def stop(self) -> int: + """The first address after the area.""" + return self.start + self.size + + def holds(self, other: Area) -> bool: + """Whether `other` lies inside this area.""" + return self.start <= other.start and other.stop <= self.stop + + def overlaps(self, other: Area) -> bool: + """Whether the two share an address.""" + return self.start < other.stop and other.start < self.stop + + +@dataclass +class Device: + """A flash device or memory region: one region and column of the drawing. `sector` is its + erase block size, when known, and `about` a header line describing where it came from. + `own_space` marks memory the CPU does not map, such as a flash chip on a SPI bus, whose + addresses are offsets in the chip.""" + + key: str + name: str + start: int + size: int + areas: list[Area] + sector: int | None = None + about: str | None = None + own_space: bool = False + + @property + def stop(self) -> int: + """The first address after the device.""" + return self.start + self.size + + +@dataclass +class Region: + """A device laid out: its blocks, as a bucket, its erase block size and its address space, + None for the CPU's, with the reason it has one.""" + + bucket: Bucket + sector: int | None + space: str | None = None + space_reason: str = "" + + +@dataclass +class Layout: + """The regions to write, what was left out and the comments saying why.""" + + regions: list[Region] + diagnostics: list[Diagnostic] + comments: list[str] + + +def _hex(value: int) -> str: + """An address for a message: 0x8000.""" + return f"0x{value:x}" + + +def _merge_same_ranges(areas: list[Area]) -> list[Area]: + """One area per range: a partition before a container, then the first listed. The others + are named in its note.""" + kept: dict[tuple[int, int], Area] = {} + order: list[tuple[int, int]] = [] + for area in sorted(areas, key=lambda a: a.container): + key = (area.start, area.size) + if key in kept: + kept[key].aliases.append(area.name) + else: + kept[key] = area + order.append(key) + return [kept[key] for key in order] + + +def _insert(area: Area, top: list[Area]) -> Area | None: + """Put `area` in the tree under the smallest area that holds it, taking in the areas it + holds. Returns the area it partly overlaps when it does not fit, and then changes nothing.""" + siblings = top + while True: + parent = next((other for other in siblings if other.holds(area)), None) + if parent is None: + break + siblings = parent.children + inside = [other for other in siblings if area.holds(other)] + clash = next((o for o in siblings if o.overlaps(area) and o not in inside), None) + if clash is not None: + return clash + area.children += inside + area.children.sort(key=lambda a: a.start) + siblings[:] = [other for other in siblings if other not in inside] + [area] + siblings.sort(key=lambda a: a.start) + return None + + +def _named(area: Area) -> str: + """How a message names an area, with the areas merged into it: `a (and b)`.""" + if not area.aliases: + return area.where + return f"{area.where} (and {', '.join(area.aliases)}, with the same range)" + + +def nest(device: Device, region_id: str = "") -> tuple[list[Area], list[Diagnostic], list[str]]: + """The device's areas as a tree, with W114 for each partition left out and a comment for + each container left out. `region_id` is the subject of the W114s.""" + diagnostics: list[Diagnostic] = [] + comments: list[str] = [] + fitting: list[Area] = [] + for area in device.areas: + if area.size <= 0: + comments.append(f"{area.where} has size 0, so it is not drawn.") + elif area.start < device.start or area.stop > device.stop: + diagnostics.append( + Diagnostic( + "W114", + f"{area.where} ({_hex(area.start)} - {_hex(area.stop - 1)}) lies outside " + f"{device.name} ({_hex(device.start)} - {_hex(device.stop - 1)}), left out", + hint="check the partition's reg against the size of its flash", + subject=region_id or None, + ) + ) + else: + fitting.append(area) + merged = _merge_same_ranges(fitting) + top: list[Area] = [] + # Partitions first, then containers, then the weak partitions, which give way to both. + order = sorted(merged, key=lambda a: (a.weak is not None, a.container, -a.size, a.start)) + for area in order: + clash = _insert(area, top) + if clash is None: + continue + what = ( + f"{_named(area)} ({_hex(area.start)} - {_hex(area.stop - 1)}) partly overlaps " + f"{clash.where} ({_hex(clash.start)} - {_hex(clash.stop - 1)})" + ) + if area.container and area.weak is None: + comments.append(f"{what}, so it is not drawn.") + else: + diagnostics.append( + Diagnostic( + "W114", + f"{what}, left out", + hint=area.weak + or "memmap draws one area inside another, but not two that partly overlap", + subject=region_id or None, + ) + ) + return top, diagnostics, comments + + +def _note(area: Area) -> str | None: + """The text beside a block: its role, then the areas with the same range.""" + parts = [area.note] if area.note else [] + if area.aliases: + parts.append("also " + ", ".join(area.aliases)) + return "; ".join(parts) or None + + +def _items(areas: list[Area], start: int, stop: int, taken: set[str]) -> list[Item]: + """Blocks for `areas` between `start` and `stop`, with Free blocks in the gaps.""" + items: list[Item] = [] + cursor = start + for area in [*areas, None]: + until = area.start if area is not None else stop + if until > cursor: + name = f"Free\n{floor_size(until - cursor)}" + items.append(Item(unique_id("free", taken, "free"), name, cursor, until - cursor, free=True)) + if area is None: + break + item = Item(unique_id(area.key, taken, "partition"), area.name, area.start, area.size, _note(area)) + if area.children: + item.children = _items(area.children, area.start, area.stop, taken) + item.expanded = True + items.append(item) + cursor = area.stop + return items + + +def lay_out(devices: list[Device]) -> Layout: + """Each device as a region of blocks, in the order given.""" + taken: set[str] = set() + region_ids = [unique_id(device.key, taken, "flash") for device in devices] + regions: list[Region] = [] + diagnostics: list[Diagnostic] = [] + comments: list[str] = [] + mapped: list[Device] = [] + for device, region_id in zip(devices, region_ids): + if device.size <= 0: + raise ImportProblem(f"{device.name} has size 0, which memmap cannot draw") + space, reason = (region_id, "offsets in the chip, not CPU addresses") if device.own_space else (None, "") + if space is None and any(device.start < other.stop and other.start < device.stop for other in mapped): + space, reason = region_id, "overlaps another region" + comments.append(f"{device.name} overlaps another region, so it gets an address space of its own.") + if space is None: + mapped.append(device) + tree, found, notes = nest(device, region_id) + diagnostics += found + comments += notes + items = _items(tree, device.start, device.stop, taken) + used = sum(item.size for item in items if not item.free) + memory = MemorySpec(region_id, device.start, device.size, device.name) + sector = device.sector + if sector is not None and (device.size % sector or device.size // sector > MAX_SECTORS): + comments.append( + f"{device.name} has no sectors: its {approx_size(sector)} erase blocks do not divide it " + f"or are more than {MAX_SECTORS}, so partition edges are not checked against them." + ) + sector = None + regions.append(Region(Bucket(memory, items, used), sector, space, reason)) + return Layout(regions, diagnostics, comments) + diff --git a/memmap/from_partitions/dts.py b/memmap/from_partitions/dts.py new file mode 100644 index 0000000..74ba09e --- /dev/null +++ b/memmap/from_partitions/dts.py @@ -0,0 +1,402 @@ +"""Read a devicetree source file, in the subset that Zephyr's merged `zephyr.dts` uses. + +The build writes `build/zephyr/zephyr.dts` after it has run the C preprocessor and merged the +board files and overlays, so the file has no `#include`, macros or expressions: nodes with +labels, and properties holding cell lists, strings, byte strings and references. That is what +this reads. A source `.dts` or overlay still needs the preprocessor, so it is refused with a +hint to use the build's file instead. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from ..from_elf.regions import ImportProblem + +MERGED_HINT = "pass the merged build/zephyr/zephyr.dts that the build writes" + +_TOKEN = re.compile( + r""" + (?P\s+) + | (?P/\*.*?\*/|//[^\n]*) + | (?P"(?:[^"\\\n]|\\.)*") + | (?P'(?:[^'\\\n]|\\.)+') + | (?P/[a-z][a-z0-9-]*/) + | (?P&\{[^}]*\}|&[A-Za-z_][A-Za-z0-9_]*) + | (?P