From 604760c2577d8f0263b006b257404f813dfa25d6 Mon Sep 17 00:00:00 2001 From: iharhl Date: Wed, 4 Nov 2026 21:33:45 +0100 Subject: [PATCH] Read memory regions from linker maps in from-elf `memmap from-elf firmware.elf --map firmware.map` takes the memory regions from the map file the linker writes, telling the format apart by its content: the Memory Configuration table of GNU ld, the execution regions of armlink (Keil), and the placement summary of IAR ILINK, whose unnamed ranges are called ROM, RAM or Memory by what is placed there. A --region with the id of a map region replaces it, and one with a new id adds a region. A map region that holds no section stays in memory without a column. memmap reads maps, not linker scripts, scatter files or .icf files, as a map holds what the linker worked out. When the map gives no region to draw, as an lld map, a GNU ld map without MEMORY or an armlink map without a scatter file does, and no --region stands in, from-elf writes nothing and exits 2. Its hint names the --region specs to give, and failures from from-elf now carry such hints in --json. Map regions left out, for no size, size 0, running past the address space, overlapping a --region or another map region, or being too small for a section, are noted in the config's header. New warning W112 reports each one whose loss changes the drawing, with the --region to give instead. A section across the edge of a map region leaves that region out rather than stopping the import. The from_elf module becomes a package, with the map readers in from_elf/linker. docs/elf-testing.md becomes docs/from-elf.md, which covers each toolchain's map, the limits, future ideas such as Rust and lld, and the tests. Tests: the readers on excerpts of 13 real GNU ld, armlink and IAR maps, against tests/golden/linker/regions.json; the fixture firmware with its own GNU ld map, which build.sh now writes; and synthetic cases for lld maps, missing sizes, overlaps, address-space edges and files that are not maps. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 45 +- README.md | 45 +- docs/elf-testing.md | 121 --- docs/from-elf.md | 345 ++++++++ memmap/cli.py | 84 +- memmap/from_elf.py | 530 ------------- memmap/from_elf/__init__.py | 30 + memmap/from_elf/blocks.py | 43 + memmap/{ => from_elf}/elf.py | 0 memmap/from_elf/importer.py | 317 ++++++++ memmap/from_elf/linker/__init__.py | 17 + memmap/from_elf/linker/armlink_map.py | 39 + memmap/from_elf/linker/files.py | 52 ++ memmap/from_elf/linker/gnu_map.py | 46 ++ memmap/from_elf/linker/iar_map.py | 120 +++ memmap/from_elf/linker/model.py | 27 + memmap/from_elf/linker/resolve.py | 181 +++++ memmap/from_elf/regions.py | 129 +++ memmap/from_elf/sizes.py | 48 ++ memmap/from_elf/writer.py | 167 ++++ tests/elf_builder.py | 23 +- tests/fixtures/elf_snapshots/refresh.py | 4 +- tests/fixtures/elf_src/build.sh | 5 +- tests/fixtures/firmware.map | 86 ++ .../linker_snapshots/iar6_stm32f103.map | 12 + .../fixtures/linker_snapshots/iar8_rt685.map | 32 + .../linker_snapshots/iar8_stm32f030.map | 16 + .../linker_snapshots/iar8_stm32f407_ccm.map | 17 + .../linker_snapshots/iar9_stm32f407.map | 24 + tests/fixtures/linker_snapshots/iar_stm8.map | 44 ++ tests/fixtures/linker_snapshots/index.json | 93 +++ .../linker_snapshots/keil_f103_ac505.map | 6 + .../linker_snapshots/keil_f103_ac506.map | 6 + .../linker_snapshots/keil_f407_compressed.map | 7 + .../linker_snapshots/keil_no_scatter.map | 7 + tests/fixtures/linker_snapshots/refresh.py | 185 +++++ .../linker_snapshots/zephyr_esp32.map | 13 + .../linker_snapshots/zephyr_nrf52dk.map | 9 + .../linker_snapshots/zephyr_qemu_x86.map | 8 + tests/golden/linker/regions.json | 247 ++++++ tests/test_elf_snapshots.py | 27 +- tests/test_from_elf.py | 2 +- tests/test_linker.py | 736 ++++++++++++++++++ 43 files changed, 3255 insertions(+), 740 deletions(-) delete mode 100644 docs/elf-testing.md create mode 100644 docs/from-elf.md delete mode 100644 memmap/from_elf.py create mode 100644 memmap/from_elf/__init__.py create mode 100644 memmap/from_elf/blocks.py rename memmap/{ => from_elf}/elf.py (100%) create mode 100644 memmap/from_elf/importer.py create mode 100644 memmap/from_elf/linker/__init__.py create mode 100644 memmap/from_elf/linker/armlink_map.py create mode 100644 memmap/from_elf/linker/files.py create mode 100644 memmap/from_elf/linker/gnu_map.py create mode 100644 memmap/from_elf/linker/iar_map.py create mode 100644 memmap/from_elf/linker/model.py create mode 100644 memmap/from_elf/linker/resolve.py create mode 100644 memmap/from_elf/regions.py create mode 100644 memmap/from_elf/sizes.py create mode 100644 memmap/from_elf/writer.py create mode 100644 tests/fixtures/firmware.map create mode 100644 tests/fixtures/linker_snapshots/iar6_stm32f103.map create mode 100644 tests/fixtures/linker_snapshots/iar8_rt685.map create mode 100644 tests/fixtures/linker_snapshots/iar8_stm32f030.map create mode 100644 tests/fixtures/linker_snapshots/iar8_stm32f407_ccm.map create mode 100644 tests/fixtures/linker_snapshots/iar9_stm32f407.map create mode 100644 tests/fixtures/linker_snapshots/iar_stm8.map create mode 100644 tests/fixtures/linker_snapshots/index.json create mode 100644 tests/fixtures/linker_snapshots/keil_f103_ac505.map create mode 100644 tests/fixtures/linker_snapshots/keil_f103_ac506.map create mode 100644 tests/fixtures/linker_snapshots/keil_f407_compressed.map create mode 100644 tests/fixtures/linker_snapshots/keil_no_scatter.map create mode 100644 tests/fixtures/linker_snapshots/refresh.py create mode 100644 tests/fixtures/linker_snapshots/zephyr_esp32.map create mode 100644 tests/fixtures/linker_snapshots/zephyr_nrf52dk.map create mode 100644 tests/fixtures/linker_snapshots/zephyr_qemu_x86.map create mode 100644 tests/golden/linker/regions.json create mode 100644 tests/test_linker.py diff --git a/AGENTS.md b/AGENTS.md index 24196cd..538d834 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,13 +15,36 @@ Run commands as `memmap ...` when it is installed, or `uv run memmap ...` inside 5. `memmap render layout.yaml -o layout.svg --json --strict` ## Typical task: map a linked firmware image -1. Read `ORIGIN` and `LENGTH` of each region in the linker script's `MEMORY` block, or in the - `Memory Configuration` table of a GNU ld `.map` file. -2. `memmap from-elf firmware.elf -o layout.yaml --region flash=0x08000000:64K - --region ram=0x20000000:20K --json` -> writes the config, then checks it like `check`. - Fix W110 (a section outside every region) by correcting or adding a `--region`. -3. Continue from step 4 above. Rename or merge blocks in `layout.yaml` as needed. `--force` - overwrites an existing output. +1. The ELF does not say how big flash and RAM are. Find the map file the linker wrote: + + | Toolchain | Map | Found in | + | --- | --- | --- | + | GNU ld: gcc, Zephyr, ESP-IDF, Pico SDK | `firmware.map` (`-Wl,-Map=`) | next to the ELF | + | Arm armlink: Keil MDK | `project.map` | `Listings/` | + | IAR ILINK | `project.map` | `List/` | + + lld (Rust, LLVM Embedded) maps list no regions, and memmap does not read linker scripts: + give each region with `--region flash=0x08000000:64K`, copying `ORIGIN` and `LENGTH` from + the script's `MEMORY` block. +2. `memmap from-elf firmware.elf -o layout.yaml --map firmware.map --json --strict` -> writes + the config, then checks it like `check`. A `--region` with the id of a map region replaces + it, and one with a new id adds a region. To split a map region, give one part its id. +3. When the map gives no region it can draw, `from-elf` writes nothing and exits 2 with E090. + Its `hint` names the `--region` specs to give, such as `--region er_ro=0x08000000:SIZE`: + fill in the sizes and run it again. +4. Fix W112 (a map region left out) and W110 (a section outside every region) by their hints: + give the `--region` the hint names, check that the map belongs to this ELF, or add a + `--region` that covers the section. +5. Continue from step 4 of the first task. Rename or merge blocks in `layout.yaml` as needed. + `--force` overwrites an existing output. + +Regions take their names from the map: FLASH gets the id `flash`. IAR maps do not name their +regions, so they are called ROM, RAM or Memory by what is placed there, numbered in address +order (`ram`, `ram_2`). A region that holds no section stays in `memory` without a column. One +without a size, of size 0, past the address space, overlapping a lower or larger region, or +too small for a section is left out, with W112 when the drawing loses something. Comments in +`layout.yaml` list both. https://github.com/iharhl/memmap/blob/main/docs/from-elf.md has the +details and limits. In a region of more than 8 blocks, small sections fold into groups such as `small_sections`, which keep the sections as `children` and each get a detail column. Zooms go one level deep, so @@ -108,8 +131,9 @@ Every diagnostic has a stable code, a YAML path, a line and usually a `hint`. In | W104 | Sector list does not add up to the region size | | W105 | Block name was split inside a word to fit the column | | W106 | Text overlaps other text, or is wider than its block | -| W110 | `from-elf`: section outside every `--region`, placed at the top level | +| W110 | `from-elf`: section outside every memory region, placed at the top level | | W111 | `from-elf`: section overlaps another one, left out | +| W112 | `from-elf`: map region left out (no size, past the address space, overlap, too small) | | E090 | Usage or IO failure, such as a missing file or bad `--set` (`--json` only) | | E091 | Render or internal failure, such as missing cairo (`--json` only) | @@ -121,7 +145,8 @@ Exit codes: `0` ok, `1` validation errors, `2` usage or IO error, `3` render err - After an intended drawing change, review the SVG diff, then run `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_golden.py`. - After an intended change to `from-elf` or the drawing, also review and regenerate the - real-firmware goldens: `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py`. - docs/elf-testing.md describes them and how to add an image. + real-firmware goldens: `MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py + tests/test_linker.py`. docs/from-elf.md describes them and how to add an image or a + map. - A new diagnostic code or recipe goes in this file too. `memmap guide` ships it to users, and `tests/test_cli.py` covers the recipes. diff --git a/README.md b/README.md index 63e6672..2c9cd82 100644 --- a/README.md +++ b/README.md @@ -66,37 +66,30 @@ uv run memmap render layout.yaml -o layout.svg | `memmap render CONFIG [-o OUT] [--format svg\|png\|pdf] [--json] [--strict] [--set key=value]` | Validate, then draw; `--strict` fails on warnings | | `memmap check CONFIG [--json] [--strict]` | Validate only; `--strict` fails on warnings | | `memmap describe CONFIG [--json]` | Resolved regions, blocks, labels and option origins | -| `memmap from-elf ELF [-o OUT] [--region id=start:size[:name]] [--min-size N] [--force] [--json] [--strict]` | Write a config from a linked ELF's sections, then check it | +| `memmap from-elf ELF [-o OUT] [--map FILE] [--region id=start:size[:name]] [--min-size N] [--force] [--json] [--strict]` | Write a config from a linked ELF's sections, then check it | | `memmap schema` | JSON Schema of the config | | `memmap guide` | Print the agent guide, for agents using an installed copy | ### From an ELF file -`from-elf` writes a config from the allocated sections of a linked firmware image: `.text`, -`.data`, `.bss` and so on, each at its real address and size. The ELF does not record how big -flash and RAM are, so give them with `--region`, copying `ORIGIN` and `LENGTH` from the -linker script's `MEMORY` block, or from the `Memory Configuration` table of a GNU ld map file: +`from-elf` writes a config from the sections of a linked firmware image, each at its real +address and size. The ELF does not record how big flash and RAM are, so pass the map file the +linker writes, from GNU ld, Keil (armlink) or IAR: ```bash -uv run memmap from-elf build/firmware.elf -o layout.yaml --region flash=0x08000000:64K --region ram=0x20000000:20K +uv run memmap from-elf build/firmware.elf -o layout.yaml --map build/firmware.map ``` -You get one column per region, titled with how much of it is used: +An lld map, as Rust and the LLVM Embedded Toolchain write, lists no regions. Give them with +`--region` instead, copying `ORIGIN` and `LENGTH` from the linker script's `MEMORY` block: -- Initialised data is drawn twice: in RAM where it runs, and in flash where startup code copies - it from. Each loadable segment is one flash block: `.data (load)`, with a note such as - `copied to RAM at 0x2000 0000`, or `RAM init data` when it holds several sections, as Zephyr's - do (`code and data` when it includes code). -- Gaps of at least `--min-size` (default 64 bytes) become Free blocks with their size, rounded - down and marked `≈` when not exact. -- In a region of more than 8 blocks, runs of smaller sections fold into one block that holds - them as children, and each such group gets a detail column beside it. Zooms go one level - deep: when sections outside every `--region` put the regions in a zoom of their own, their - groups list the section names in a note instead. -- Without `--region`, sections are drawn on the whole address space. Object files converted - from a `.hex` or `.bin` image, which keep their real addresses, are read too. +```bash +uv run memmap from-elf build/firmware.elf -o layout.yaml --region flash=0x08000000:64K --region ram=0x20000000:20K +``` -The output is an ordinary config: edit it, then `render` it. +The output is an ordinary config: edit it, then `render` it. [docs/from-elf.md](docs/from-elf.md) +covers where each toolchain puts its map, what is read from it, what happens when it is not +enough, and how sections become blocks. A minimal config: @@ -128,6 +121,7 @@ set it on the expansion: `expand: [{ region: flash, title: Flash }]`. ## Learn more - [AGENTS.md](AGENTS.md): task guide, recipes and the full diagnostic list. `memmap guide` prints it. +- [docs/from-elf.md](docs/from-elf.md): drawing a linked firmware image, its map readers and limits. - [examples/](examples/): see the table below. - `memmap schema`: every key with a description and an example. Point your editor's YAML extension at it for completion. @@ -177,16 +171,15 @@ regenerate them: MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_golden.py ``` -`from-elf` is also tested on eight real firmware images, kept as small header snapshots with -golden configs and drawings in `tests/golden/elf/`. After an intended change to the importer or -the drawing, review and regenerate those too: +`from-elf` is also tested on real firmware images and linker maps, kept as small snapshots with +goldens. After an intended change to the importer, the map readers or the drawing, review and +regenerate those too: ```bash -MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py tests/test_linker.py ``` -[docs/elf-testing.md](docs/elf-testing.md) explains the approach, lists the images and shows -their maps. +[docs/from-elf.md](docs/from-elf.md#testing) explains the approach and shows the drawings. Text is measured with a built-in Helvetica/Arial width table, so wrapping is slightly approximate for other fonts. diff --git a/docs/elf-testing.md b/docs/elf-testing.md deleted file mode 100644 index fe82deb..0000000 --- a/docs/elf-testing.md +++ /dev/null @@ -1,121 +0,0 @@ -# Testing the ELF import - -`memmap from-elf` makes judgement calls: which sections to draw, which to fold, where free space -goes, what to call the flash copy of `.data`. Those are easy to get wrong on layouts nobody -thought of, so besides hand-made cases the tests run real firmware through the importer. - -## Test layers - -| Layer | Where | Covers | -| --- | --- | --- | -| Synthetic ELFs | `tests/test_from_elf.py`, built by `tests/elf_builder.py` | One rule per test: folding, free gaps, load images, several memory banks, ids, 64-bit, big-endian, converted images, bad files, the CLI | -| Real builds | `tests/fixtures/firmware*.elf`, from `tests/fixtures/elf_src/` | The reader on files binutils wrote: one firmware as 32- and 64-bit, little- and big-endian. `build.sh` rebuilds them with `arm-none-eabi-gcc` and GNU `objcopy` | -| Real firmware | `tests/test_elf_snapshots.py`, `tests/fixtures/elf_snapshots/` | Eight images from different toolchains, OSes and architectures, against golden configs and drawings | - -## Snapshots instead of ELF files - -The real images total 6.5 MB, mostly debug info memmap never reads, and they are other projects' -builds. So the repo keeps a snapshot of each instead: the section headers, the loadable segments, -the `--region` specs to import with, and the source URL and SHA-256. That is 3 to 6 KB of JSON, -with no code or data from the firmware. - -The test rebuilds each snapshot into a minimal ELF, imports it, and checks that the import reports -nothing, that the config matches `tests/golden/elf/.yaml` and passes `check --strict`, and -that its drawing matches `tests/golden/elf/.svg`. The configs from snapshots were identical -to those from the real files when the snapshots were made. - -Snapshots cannot catch quirks in how a toolchain writes the file itself, as the rebuilt files are -clean. The real builds and the synthetic tests cover the reader. - -## The images - -All come from [Renode](https://github.com/renode/renode), whose board scripts -(`scripts/single-node/*.resc`) load prebuilt firmware from `dl.antmicro.com`. Locally they live -in `build/elf/`, which git ignores. - -| Snapshot | Firmware | Regions | -| --- | --- | --- | -| `nrf52840_zephyr_shell` | Zephyr shell, nRF52840 (Cortex-M4) | flash 1M at 0, RAM 256K at 0x20000000 | -| `r52_zephyr_philosophers` | Zephyr philosophers, Cortex-R52 | DRAM 2G at 0 | -| `fe310_zephyr_shell` | Zephyr shell, SiFive FE310 (RISC-V), runs from SPI flash | SPI flash 12M at 0x20400000, DTIM 16K at 0x80000000 | -| `stm32f412_tock_kernel` | Tock (Rust) kernel, STM32F412 | flash 1M at 0x08000000, SRAM 256K at 0x20000000 | -| `msp430f2619_hello` | Hello world, MSP430F2619 (16-bit) | flash 0x1DF00 at 0x2100, RAM 4K at 0x1100 | -| `stm32f746_dartino_lines` | Dartino (Dart VM) demo, STM32F746 | flash 1M at 0x08000000, SRAM 320K at 0x20000000 | -| `stm32f4discovery` | STM32F4 Discovery demo, STM32F407 | flash 1M at 0x08000000, SRAM 128K at 0x20000000 | -| `apollo4_hello_uart` | AmbiqSuite hello world, Ambiq Apollo4 (Cortex-M4F) | MRAM 1952K at 0x18000, TCM 384K at 0x10000000, shared SRAM 1M at 0x10060000 | - -The region sizes are the chips' real flash and RAM, from datasheets and Renode's platform files -(`platforms/cpus/*.repl`), since the ELF does not record them. Where the firmware starts above -the bottom of its memory, the region starts there too: the FE310 shell runs from the user part of -SPI flash, and the Apollo4 application starts 96 KB into its 2 MB MRAM. - -## Refreshing and adding images - -`tests/fixtures/elf_snapshots/refresh.py` lists the images with their URL, description and -`--region` specs, and rewrites every snapshot whose ELF is in `build/elf/`. `--download` fetches -the missing ones first: - -```bash -uv run python tests/fixtures/elf_snapshots/refresh.py --download -``` - -To add an image, add it to `MANIFEST` in that script and run it. Then, as after any intended -change to the importer or the drawing, regenerate the goldens and review their diff: - -```bash -MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py -``` - -## The maps - -These are the goldens, so they always show what the current code draws. - -### nRF52840, Zephyr shell - -Two folded groups, each zoomed into, and one "RAM init data" block for the seven RAM sections -that startup code fills from flash. - -![nRF52840 Zephyr shell](../tests/golden/elf/nrf52840_zephyr_shell.svg) - -### Cortex-R52, Zephyr philosophers - -Three folded groups with a detail column each, in one 2 GB memory. - -![Cortex-R52 Zephyr philosophers](../tests/golden/elf/r52_zephyr_philosophers.svg) - -### FE310, Zephyr shell - -RISC-V running from SPI flash, with data in the 16 KB DTIM. - -![FE310 Zephyr shell](../tests/golden/elf/fe310_zephyr_shell.svg) - -### STM32F412, Tock kernel - -A 1-byte `.apps` section far up flash, with free space on both sides. - -![STM32F412 Tock kernel](../tests/golden/elf/stm32f412_tock_kernel.svg) - -### Apollo4, AmbiqSuite hello world - -Three banks: code in MRAM, data in TCM and in shared SRAM. Each bank's initial data is copied -from MRAM in a segment of its own, and each copy's note names the bank it goes to. - -![Apollo4 hello world](../tests/golden/elf/apollo4_hello_uart.svg) - -### MSP430F2619, hello world - -Small sections drawn in place, and the reset vector at 0xFFFE splitting flash in two. - -![MSP430F2619 hello world](../tests/golden/elf/msp430f2619_hello.svg) - -### STM32F746, Dartino lines - -A large `.data` copied from flash, and the stack and heap reserved as a section. - -![STM32F746 Dartino lines](../tests/golden/elf/stm32f746_dartino_lines.svg) - -### STM32F4 Discovery demo - -A conventional STM32 layout. - -![STM32F4 Discovery demo](../tests/golden/elf/stm32f4discovery.svg) diff --git a/docs/from-elf.md b/docs/from-elf.md new file mode 100644 index 0000000..f2d2e4f --- /dev/null +++ b/docs/from-elf.md @@ -0,0 +1,345 @@ +# Drawing a firmware image: `memmap from-elf` + +`memmap from-elf` turns a linked firmware image into a memmap config: every allocated section +(`.text`, `.data`, `.bss` and so on) at its real address and size, placed in the chip's memory +regions. The config is then checked like `memmap check`, and you can edit and render it like +any other. + +```bash +memmap from-elf build/firmware.elf -o layout.yaml --map build/firmware.map +memmap render layout.yaml -o layout.svg +``` + +The ELF says where every section is, but not how big flash and RAM are. That comes from the +map file the linker writes, or from `--region` specs you give by hand. + +- [Where the regions come from](#where-the-regions-come-from) +- [Limits: when the map is not enough](#limits-when-the-map-is-not-enough) +- [From sections to blocks](#from-sections-to-blocks) +- [Future ideas](#future-ideas): Rust and lld, and what else is not covered yet +- [Testing](#testing) +- [The maps](#the-maps): the real-firmware goldens, drawn + +## Where the regions come from + +| Given | Regions | +| --- | --- | +| `--map FILE` | Read from the linker's map file | +| `--region ID=START:SIZE[:NAME]` | One per spec, such as `--region flash=0x08000000:64K:Flash` | +| Both | The map's, with each `--region` replacing the map region with its id, or adding one | +| Neither | None: sections are drawn on the whole address space | + +### The map, by toolchain + +| Toolchain | Turn the map on with | Usually at | What memmap reads | +| --- | --- | --- | --- | +| GNU ld: arm-none-eabi-gcc, Zephyr, ESP-IDF, Pico SDK, STM32CubeIDE | `-Wl,-Map=firmware.map` | next to the ELF | The Memory Configuration table: each `MEMORY` region with its origin and length | +| Arm armlink: Keil MDK, Arm Compiler 5 | `--map` (Keil's linker listing) | `Listings/` | Each execution region's base, and its `Max` as the size | +| IAR ILINK | `--map` (Linker > List > Generate linker map file) | `List/` | The address ranges of the placement summary | +| LLVM lld: Rust, LLVM Embedded Toolchain | `-Map`, but it lists no regions | | Nothing: give the regions with `--region` | + +Zephyr, ESP-IDF and the Pico SDK write a map by default. memmap tells the formats apart by +their content, so the file name does not matter. Maps from armlink 5.05 and 5.06 are tested; +none from the armlink of Arm Compiler 6 is yet. + +A map holds what the linker worked out: the `MEMORY` block after the preprocessor, `INCLUDE` +files and `--defsym` symbols are all applied. That is why memmap reads the map and not the +linker script, scatter file (`.sct`) or `.icf` file the linker was given. A linker script passed +to `--map` is turned away with a message saying so. + +### Names and ids + +Regions take their names from the map, and their ids from the names: `FLASH` gets the id +`flash`, `RAM_D1` gets `ram_d1`, and a repeated id gets a number, `ram_2`. + +IAR maps do not name their regions. They list the address ranges each placement went to, which +are the `.icf` file's regions, worked out. memmap merges overlapping and adjoining ranges and +calls them by what is placed there: `RAM` when read-write data or a stack goes there, `ROM` +when only read-only content does, and `Memory` when the map does not say, as for a range that +holds named sections only. A comment at the top of the config says so. Repeated names are +numbered in address order, so on an STM32F407 with CCM RAM at 0x10000000, `ram` is the CCM and +`ram_2` the main SRAM: check the ids in the output before replacing one with `--region`. + +### Mixing the map and `--region` + +A `--region` with the id of a map region replaces it, which is how you correct one size or name: + +```bash +memmap from-elf firmware.elf --map firmware.map --region flash=0x08000000:128K:Flash +``` + +A `--region` with a new id adds a region. To split a map region, give one of the parts its id, +so that it replaces the map region, and the others new ids: + +```bash +memmap from-elf firmware.elf --map firmware.map --region flash=0x08000000:16K:Boot --region app=0x08004000:240K +``` + +With new ids for both parts, `FLASH` would overlap them and be left out with W112, which +`--strict` fails on. + +### Regions left out, and regions without a column + +| Map region | What happens | W112 | +| --- | --- | --- | +| Holds no section, such as an unused CCM RAM or Zephyr's `IDT_LIST` | Stays in `memory`, without a column | No | +| No size, such as an armlink region placed without a scatter file | Left out | Yes, with the `--region` to give, unless its start lies in a drawn region. One per start | +| Size 0 | Left out | No, as it holds nothing | +| Runs past the end of the ELF's address space | Left out | Yes, unless its start lies in a drawn region | +| Overlaps a `--region` | Left out | Yes, unless it lies inside the `--region` | +| Overlaps another map region, as when a script declares one RAM twice, for its instruction and data bus | Left out, unless it starts lower than the other or, at the same start, is larger or listed first | Yes, unless it lies inside the other | +| A section runs across its edge | Left out, and the sections placed again without it | Yes, with the `--region` to give | + +W112 is reported only when the drawing loses something, and its hint says what to give +instead. Comments at the top of the config list every region left out and why, warned about or +not, and the `view` section names the regions without a column. To draw an empty region, add a +column with its id as root. + +## Limits: when the map is not enough + +> [!IMPORTANT] +> `from-elf` never makes up where a region starts or how big it is. When the map does not give +> a region it can draw, and no `--region` stands in, it writes nothing and exits 2 (E090 with +> `--json`), and the hint says which `--region` specs to give instead. When the map gives some +> regions but not the one a section is in, the region it left out gets W112 and the section +> W110, and the section is drawn outside every region. The one guess is a name: IAR maps name +> no regions, so memmap names them by what is placed there. + +| Case | What the map holds | What `from-elf` does | What to give | +| --- | --- | --- | --- | +| lld: Rust (cortex-m-rt, Embassy, Tock), LLVM Embedded Toolchain | No region table at all | Stops | A `--region` per entry of the linker script's `MEMORY` block, such as `memory.x` | +| GNU ld, with a script that has no `MEMORY` block | Only the catch-all `*default*` region | Stops | A `--region` per memory | +| armlink without a scatter file, which places regions with `--ro-base` and `--rw-base` | Each region's base, but no `Max` | Stops, suggesting `--region er_ro=0x08000000:SIZE` for each start | The suggested specs, with sizes | +| armlink, with some regions given and one still without a size | | W112 on that region, with the `--region` that is missing, and W110 on its sections | That `--region` | +| armlink, with a tight `Max` on the code region | The RW load image placed after the code, past that `Max` | W112 on the region the load image runs across, and W110 on its sections | A `--region` with that region's id, large enough to hold it | +| IAR | Only the ranges content was placed in | A section placed at a fixed address outside them, such as a boot header, gets W110 | A `--region` that covers it | +| A map from another build | Regions that do not match the sections | W110 on every section | The map of this build | + +A section that runs across the edge of a `--region` still stops the import, as only you can say +which edge is wrong. + +For example, an lld map: + +```text +$ memmap from-elf firmware.elf --map firmware.map +memmap: firmware.map is an lld map, which lists no memory regions + hint: give each region with --region ID=START:SIZE, copying ORIGIN and LENGTH from the linker script's MEMORY block +``` + +A Rust `memory.x` such as + +```text +MEMORY +{ + FLASH : ORIGIN = 0x00000000, LENGTH = 1024K + RAM : ORIGIN = 0x20000000, LENGTH = 256K +} +``` + +becomes `--region flash=0x00000000:1024K --region ram=0x20000000:256K`. Passing the lld map as +well does no harm: its regions come from `--region`, and a comment says the map listed none. + +W110 and W112 are warnings, so `from-elf` still writes the config and exits 0. Use `--strict` +to fail on them, as CI should. + +Other things a map does not tell memmap: + +- **Sub-regions.** Regions are flat. A bootloader and an application slot inside one `FLASH` + region are one column; split it with `--region` as above, or nest blocks in the config + afterwards. +- **What a region is for.** Names come from the map as written. Rename them in the config. +- **Overlays.** Of sections that share addresses, such as overlays and some `NOLOAD` sections, + one is drawn; each that overlaps it gets W111 and is left out. + +## From sections to blocks + +- **Which sections.** Every allocated section with a size. The thread-local `.tbss` is skipped, + as it only holds a template. +- **Load images.** Initialised data is drawn twice: in RAM where it runs, and in flash where + startup code copies it from. Each loadable segment stored away from where it runs is one flash + block: `.data (load)` with a note such as `copied to RAM at 0x2000 0000`, or `RAM init data` + when it holds several sections, as Zephyr's do, and `code and data` when it includes code. +- **Free space.** Gaps of at least `--min-size` (default 64 bytes) become Free blocks with their + size, rounded down and marked `≈` when not exact. Smaller gaps are drawn as a split edge. +- **Folding.** In a region of more than 8 blocks, runs of sections smaller than `--min-size` + fold into one group, such as `5 small sections`, which keeps them as `children` and gets a + detail column beside it. `--min-size 0` turns folding off. +- **Titles.** Each region's column is titled with its name and how much of it the sections + use, such as `1.6 KB of 64 KB (3%)`. +- **Sections outside every region.** They are placed at the top level with W110, and the + drawing becomes one column for the whole address space with each region zoomed out of it. + Zooms go one level deep, so the regions' groups then list their sections in a note instead + of a detail column. +- **No regions at all.** Sections are drawn on the whole address space. Object files converted + from a `.hex` or `.bin` image, which keep their real addresses, are read too. + +The output is an ordinary config. Rename, merge or annotate blocks freely, then `render` it. + +## Future ideas + +Not built yet, roughly in order of how many people they would help: + +- **Rust and lld.** lld maps list no regions, so Rust firmware (cortex-m-rt, Embassy, RTIC, + Tock) and the LLVM Embedded Toolchain need a `--region` per memory today. Nearly every Rust + `memory.x` is a plain `MEMORY` block of literal origins and lengths, such as + `FLASH : ORIGIN = 0x00000000, LENGTH = 1024K`. A reader for that alone could take regions from + `memory.x` and turn away anything else (symbols, arithmetic, `INCLUDE`) with the `--region` + to give instead, keeping the rule that memmap never works a region out itself. Another way in + is the ELF: cortex-m-rt defines symbols such as `_stack_start` at the top of RAM, which give + one edge of a region but not both. +- **armlink load regions.** A load region's `Max` bounds the whole flash image, including RW + load data that runs past a tight execution region. Reading it would draw that image inside a + region instead of leaving the region out. +- **Arm Compiler 6.** Its armlink is expected to write the same execution region lines as 5.06, + but no AC6 map is tested yet. +- **Stable IAR ids.** Numbering repeated IAR names by size, the largest first, would make `ram` + the main SRAM rather than whichever RAM range has the lowest address. +- **Splitting instead of dropping.** A `--region` inside a map region could cut the map region + around it rather than leave it out, at the cost of drawing parts the map never named. + +## Testing + +`from-elf` makes judgement calls: which sections to draw, which to fold, where free space goes, +what to call the flash copy of `.data`, which map region to keep. Those are easy to get wrong +on layouts nobody thought of, so besides hand-made cases the tests run real firmware and real +maps through the importer. + +| Layer | Where | Covers | +| --- | --- | --- | +| Synthetic ELFs | `tests/test_from_elf.py`, built by `tests/elf_builder.py` | One rule per test: folding, free gaps, load images, several memory banks, ids, 64-bit, big-endian, converted images, bad files, the CLI | +| Real builds | `tests/fixtures/firmware*.elf`, from `tests/fixtures/elf_src/` | The reader on files binutils wrote: one firmware as 32- and 64-bit, little- and big-endian, and its GNU ld map. `build.sh` rebuilds them with `arm-none-eabi-gcc` and GNU `objcopy` | +| Real firmware | `tests/test_elf_snapshots.py`, `tests/fixtures/elf_snapshots/` | Eight images from different toolchains, OSes and architectures, against golden configs and drawings | +| Linker maps | `tests/test_linker.py`, `tests/fixtures/linker_snapshots/` | The map readers on 13 real maps, against `tests/golden/linker/regions.json`, and every case of [the limits](#limits-when-the-map-is-not-enough): lld maps, maps without `MEMORY` or sizes, overlapping regions, sections across a region's edge, files that are not maps | + +### Snapshots instead of ELF files + +The real images total 6.5 MB, mostly debug info memmap never reads, and they are other projects' +builds. So the repo keeps a snapshot of each instead: the section headers, the loadable segments, +the `--region` specs to import with, and the source URL and SHA-256. That is 3 to 6 KB of JSON, +with no code or data from the firmware. + +The test rebuilds each snapshot into a minimal ELF, imports it, and checks that the import reports +nothing, that the config matches `tests/golden/elf/.yaml` and passes `check --strict`, and +that its drawing matches `tests/golden/elf/.svg`. The configs from snapshots were identical +to those from the real files when the snapshots were made. + +Snapshots cannot catch quirks in how a toolchain writes the file itself, as the rebuilt files are +clean. The real builds and the synthetic tests cover the reader. + +### The images + +All come from [Renode](https://github.com/renode/renode), whose board scripts +(`scripts/single-node/*.resc`) load prebuilt firmware from `dl.antmicro.com`. Locally they live +in `build/elf/`, which git ignores. + +| Snapshot | Firmware | Regions | +| --- | --- | --- | +| `nrf52840_zephyr_shell` | Zephyr shell, nRF52840 (Cortex-M4) | flash 1M at 0, RAM 256K at 0x20000000 | +| `r52_zephyr_philosophers` | Zephyr philosophers, Cortex-R52 | DRAM 2G at 0 | +| `fe310_zephyr_shell` | Zephyr shell, SiFive FE310 (RISC-V), runs from SPI flash | SPI flash 12M at 0x20400000, DTIM 16K at 0x80000000 | +| `stm32f412_tock_kernel` | Tock (Rust) kernel, STM32F412 | flash 1M at 0x08000000, SRAM 256K at 0x20000000 | +| `msp430f2619_hello` | Hello world, MSP430F2619 (16-bit) | flash 0x1DF00 at 0x2100, RAM 4K at 0x1100 | +| `stm32f746_dartino_lines` | Dartino (Dart VM) demo, STM32F746 | flash 1M at 0x08000000, SRAM 320K at 0x20000000 | +| `stm32f4discovery` | STM32F4 Discovery demo, STM32F407 | flash 1M at 0x08000000, SRAM 128K at 0x20000000 | +| `apollo4_hello_uart` | AmbiqSuite hello world, Ambiq Apollo4 (Cortex-M4F) | MRAM 1952K at 0x18000, TCM 384K at 0x10000000, shared SRAM 1M at 0x10060000 | + +Only the ELF files are published, so the regions are `--region` specs: the chips' real +flash and RAM, from datasheets and Renode's platform files (`platforms/cpus/*.repl`). Where the +firmware starts above the bottom of its memory, the region starts there too: the FE310 shell +runs from the user part of SPI flash, and the Apollo4 application starts 96 KB into its 2 MB +MRAM. The Tock kernel is linked by lld, so its regions would have to be given this way anyway. + +To refresh the snapshots or add an image, see `tests/fixtures/elf_snapshots/refresh.py`. It +lists the images with their URL, description and `--region` specs, and rewrites every snapshot +whose ELF is in `build/elf/`; `--download` fetches the missing ones first. Then, as after any +intended change to the importer or the drawing, regenerate the goldens and review their diff: + +```bash +uv run python tests/fixtures/elf_snapshots/refresh.py --download +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_elf_snapshots.py +``` + +### Linker maps + +The map readers are tested on real maps from public projects, chosen for the format variants +they show: + +| Snapshots | Toolchain | Shows | +| --- | --- | --- | +| `zephyr_*.map` | GNU ld | Three Zephyr builds (nRF52832, ESP32, x86): Windows line ends, 64-bit wide addresses for a 32-bit target, Zephyr's `IDT_LIST` region | +| `keil_*.map` | armlink | Arm Compiler 5.05 `Base:` and 5.06 `Exec base:`, compressed RW data, an empty CCM region, regions with no `Max` | +| `iar*.map` | IAR ILINK | Versions 6 to 9, `'` digit groups, unions of ranges, `[0x0-0xff]` ranges on STM8, EEPROM with named sections only | + +lld's map lists no regions, so it has no snapshot; the tests check that it is told apart and +that the import then asks for `--region`. The fixture firmware in `tests/fixtures/` has its GNU +ld map, which `build.sh` writes, and `--map` and the matching `--region` specs give the same +config. + +A snapshot keeps only the lines the reader uses: the Memory Configuration table, the region +lines of an armlink map, the placement summary of an IAR map. Each starts with its source URL, +pinned to a commit, and `index.json` records the SHA-256 of each full file. +`tests/fixtures/linker_snapshots/refresh.py` rewrites them from the full files in +`build/linker/`, `--download` fetching the missing ones first. Where the full files are there, +the tests also check that each reads to the same regions as its snapshot. + +To add a map, add it to `MANIFEST` in that script, run it, then regenerate the expected regions +and review their diff: + +```bash +uv run python tests/fixtures/linker_snapshots/refresh.py --download +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_linker.py +``` + +## The maps + +These are the real-firmware goldens, so they always show what the current code draws. + +### nRF52840, Zephyr shell + +Two folded groups, each zoomed into, and one "RAM init data" block for the seven RAM sections +that startup code fills from flash. + +![nRF52840 Zephyr shell](../tests/golden/elf/nrf52840_zephyr_shell.svg) + +### Cortex-R52, Zephyr philosophers + +Three folded groups with a detail column each, in one 2 GB memory. + +![Cortex-R52 Zephyr philosophers](../tests/golden/elf/r52_zephyr_philosophers.svg) + +### FE310, Zephyr shell + +RISC-V running from SPI flash, with data in the 16 KB DTIM. + +![FE310 Zephyr shell](../tests/golden/elf/fe310_zephyr_shell.svg) + +### STM32F412, Tock kernel + +A 1-byte `.apps` section far up flash, with free space on both sides. + +![STM32F412 Tock kernel](../tests/golden/elf/stm32f412_tock_kernel.svg) + +### Apollo4, AmbiqSuite hello world + +Three banks: code in MRAM, data in TCM and in shared SRAM. Each bank's initial data is copied +from MRAM in a segment of its own, and each copy's note names the bank it goes to. + +![Apollo4 hello world](../tests/golden/elf/apollo4_hello_uart.svg) + +### MSP430F2619, hello world + +Small sections drawn in place, and the reset vector at 0xFFFE splitting flash in two. + +![MSP430F2619 hello world](../tests/golden/elf/msp430f2619_hello.svg) + +### STM32F746, Dartino lines + +A large `.data` copied from flash, and the stack and heap reserved as a section. + +![STM32F746 Dartino lines](../tests/golden/elf/stm32f746_dartino_lines.svg) + +### STM32F4 Discovery demo + +A conventional STM32 layout. + +![STM32F4 Discovery demo](../tests/golden/elf/stm32f4discovery.svg) diff --git a/memmap/cli.py b/memmap/cli.py index 6bc03e3..aec7b07 100644 --- a/memmap/cli.py +++ b/memmap/cli.py @@ -14,8 +14,17 @@ from .config import ConfigError from .describe import describe_data, describe_text from .diagnostics import Diagnostic -from .elf import ElfError, read_elf -from .from_elf import ImportProblem, import_elf, parse_region_spec +from .from_elf import ( + ElfError, + Imported, + ImportProblem, + Resolved, + import_elf, + parse_region_spec, + read_elf, + read_map, + resolve_memories, +) from .pipeline import Result, run from .render.raster import RasterUnavailable, convert from .render.svg import render_svg @@ -76,16 +85,22 @@ class OutputFormat(str, Enum): def _fail( - message: str, code: int, as_json: bool = False, result: Result | None = None + message: str, + code: int, + as_json: bool = False, + result: Result | None = None, + hint: str | None = None, ) -> Exit: """Report a failure and return the Exit to raise. With `as_json` the message joins - `result`'s diagnostics, under the code FAILURE_CODES gives for `code`.""" + `result`'s diagnostics, under the code FAILURE_CODES gives for `code`, with `hint`.""" if as_json: result = result or Result() - result.diagnostics.append(Diagnostic(FAILURE_CODES[code], message)) + result.diagnostics.append(Diagnostic(FAILURE_CODES[code], message, hint=hint)) _print_json(False, result) else: click.echo(f"memmap: {message}", err=True) + if hint: + click.echo(f" hint: {hint}", err=True) return Exit(code) @@ -243,23 +258,48 @@ def describe(config: Path, as_json: bool, sets: tuple[str, ...]) -> None: click.echo(describe_text(result)) +def _import( + elf: Path, map_file: Path | None, specs: tuple[str, ...], min_size: int +) -> tuple[Imported, list[Diagnostic]]: + """Read the --region specs, the map and the ELF, in that order, and import them. Returns + the import and the map's own warnings. Raises ImportProblem with the message to print.""" + memories = [parse_region_spec(spec) for spec in specs] + linker = read_map(map_file) if map_file else None + try: + image = read_elf(elf.read_bytes()) + except OSError as exc: + raise ImportProblem(f"cannot read {elf}: {exc}") from None + except ElfError as exc: + raise ImportProblem(f"{elf}: {exc}") from None + resolved = resolve_memories(linker, memories, image.bits) if linker else Resolved(memories) + try: + imported = import_elf(image, elf.name, resolved.memories, min_size, tuple(resolved.notes)) + except ImportProblem as exc: # about the ELF and its regions + raise ImportProblem(f"{elf}: {exc}", exc.hint) from None + return imported, resolved.diagnostics + + @app.command( name="from-elf", - epilog=( - "\b\nExample:\n memmap from-elf firmware.elf -o layout.yaml " - "--region flash=0x08000000:64K --region ram=0x20000000:20K" - ), + epilog="\b\nExample:\n memmap from-elf firmware.elf -o layout.yaml --map firmware.map", ) @click.argument("elf", type=click.Path(path_type=Path)) @click.option( "-o", "--output", type=click.Path(path_type=Path), help="Config to write. Default: ELF.yaml." ) +@click.option( + "--map", + "map_file", + type=click.Path(path_type=Path), + help="Read the memory regions from a GNU ld, armlink (Keil) or IAR map file.", +) @click.option( "--region", "regions", multiple=True, metavar="ID=START:SIZE[:NAME]", - help="A memory region, such as flash=0x08000000:64K:Flash (repeatable).", + help="A memory region, such as flash=0x08000000:64K:Flash (repeatable). With --map, it " + "replaces the map's region with the same id, or adds one.", ) @click.option( "--min-size", @@ -273,6 +313,7 @@ def describe(config: Path, as_json: bool, sets: tuple[str, ...]) -> None: def from_elf( elf: Path, output: Path | None, + map_file: Path | None, regions: tuple[str, ...], min_size: str, force: bool, @@ -281,9 +322,10 @@ def from_elf( ) -> None: """Write a config from the allocated sections of an ELF file. - ELF is a linked executable. Give each memory region with --region, as the ELF does not say - how big flash and RAM are; without any, sections are drawn on the whole address space. The - written config is then checked like `memmap check`. + ELF is a linked executable. The ELF does not say how big flash and RAM are, so give the + linker's map with --map, or each region with --region, as for lld, whose map lists none. + Without either, sections are drawn on the whole address space. The written config is then + checked like `memmap check`. """ target = output or elf.with_suffix(".yaml") if target.resolve() == elf.resolve(): @@ -295,20 +337,14 @@ def from_elf( except ValueError as exc: raise _fail(f"bad --min-size: {exc}", EXIT_USAGE, as_json) try: - memories = [parse_region_spec(spec) for spec in regions] + imported, warnings = _import(elf, map_file, regions, threshold) except ImportProblem as exc: - raise _fail(str(exc), EXIT_USAGE, as_json) - try: - data = elf.read_bytes() - except OSError as exc: - raise _fail(f"cannot read {elf}: {exc}", EXIT_USAGE, as_json) - try: - imported = import_elf(read_elf(data), elf.name, memories, threshold) - except (ElfError, ImportProblem) as exc: - raise _fail(f"{elf}: {exc}", EXIT_USAGE, as_json) + raise _fail(str(exc), EXIT_USAGE, as_json, hint=exc.hint) + except Exception as exc: # noqa: BLE001 # odd input must still exit 3 with a message + raise _fail(f"internal error: {exc}", EXIT_RENDER, as_json) _write(target, imported.text.encode("utf-8"), as_json) result = _load(target, (), as_json=as_json) - result.diagnostics[:0] = imported.diagnostics + result.diagnostics[:0] = [*warnings, *imported.diagnostics] ok = _passes(result, strict) if as_json: _print_json(ok, result, output=str(target)) diff --git a/memmap/from_elf.py b/memmap/from_elf.py deleted file mode 100644 index 3aa96b1..0000000 --- a/memmap/from_elf.py +++ /dev/null @@ -1,530 +0,0 @@ -"""Turn an ELF file into a memmap config: its allocated sections, placed in memory regions. - -The ELF says where each section runs and where its bytes are loaded from, but not how big flash -and RAM are; those come from `--region` specs. The result is ordinary YAML for the user to edit. -""" - -from __future__ import annotations - -import itertools -import json -import math -import re -from dataclasses import dataclass, field - -from .diagnostics import Diagnostic -from .elf import PT_LOAD, SHF_TLS, Elf, Section -from .units import diagnostic_address, format_address, parse_address, parse_size - -_ID_RE = re.compile(r"^[a-z0-9_]+$") -_NOTE_NAMES = 6 # section names a note lists before "+N more" -FOLD_OVER = 8 # a region with more blocks than this folds its runs of small sections -_SHF_EXECINSTR = 0x4 -COLUMN_WIDTH = 150 # px, room for section names such as .ARM.exidx and the usage title -_UNITS = ((1024**3, "GB"), (1024**2, "MB"), (1024, "KB")) - - -class ImportProblem(ValueError): - """The ELF and the `--region` specs do not fit together, or a spec is malformed.""" - - -@dataclass(frozen=True) -class MemorySpec: - """One memory region from `--region`: an id, a range and a display name.""" - - id: str - start: int - size: int - name: str - - @property - def end(self) -> int: - """Last address, inclusive.""" - return self.start + self.size - 1 - - -def parse_region_spec(text: str) -> MemorySpec: - """Parse `id=start:size[:name]`, such as `flash=0x08000000:64K:Internal flash`.""" - region_id, separator, rest = text.partition("=") - parts = rest.split(":", 2) - if not separator or len(parts) < 2: - raise ImportProblem(f"bad --region {text!r}, expected id=start:size[:name]") - region_id = region_id.strip() - if not _ID_RE.match(region_id): - raise ImportProblem(f"bad --region id {region_id!r}, use lowercase letters, digits and _") - try: - start, size = parse_address(parts[0]), parse_size(parts[1]) - except ValueError as exc: - raise ImportProblem(f"bad --region {text!r}: {exc}") from None - name = parts[2].strip() if len(parts) == 3 and parts[2].strip() else region_id.upper() - return MemorySpec(region_id, start, size, name) - - -@dataclass -class _Item: - """A block to write: a section, a load image, a folded group of small sections, or free - space. A group holds its sections as `children`; `expanded` gives it a detail column.""" - - id: str - name: str - start: int - size: int - note: str | None = None - free: bool = False - children: list[_Item] = field(default_factory=list) - expanded: bool = False - - @property - def end(self) -> int: - """Last address, inclusive.""" - return self.start + self.size - 1 - - -@dataclass -class _Bucket: - """The items placed in one memory region, or at the top level when `memory` is None.""" - - memory: MemorySpec | None - items: list[_Item] = field(default_factory=list) - - -@dataclass -class Imported: - """The generated YAML and what the import had to say about the ELF.""" - - text: str - diagnostics: list[Diagnostic] - - -class _Ids: - """Hands out unique region ids derived from section names.""" - - def __init__(self, taken: list[str]) -> None: - """Start with the ids that are already in use.""" - self.taken = set(taken) - - def make(self, text: str) -> str: - """A fresh id from `text`: lowercase, non-alphanumerics as _, suffixed when taken.""" - base = re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") or "section" - candidate, number = base, 2 - while candidate in self.taken: - candidate, number = f"{base}_{number}", number + 1 - self.taken.add(candidate) - return candidate - - -def _check_memories(memories: list[MemorySpec]) -> None: - """Reject duplicate ids and overlapping regions among the `--region` specs.""" - seen: set[str] = set() - for memory in memories: - if memory.id in seen: - raise ImportProblem(f"--region {memory.id} is given twice") - seen.add(memory.id) - ordered = sorted(memories, key=lambda memory: memory.start) - for low, high in itertools.pairwise(ordered): - if high.start <= low.end: - raise ImportProblem(f"--region {low.id} and {high.id} overlap") - - -def _wanted(section: Section) -> bool: - """Whether a section takes space on the target. Thread-local .tbss only has a template.""" - if not section.allocated or section.size == 0: - return False - return not (section.nobits and section.flags & SHF_TLS) - - -def _approx_size(size: int) -> str: - """A size rounded to one decimal, for usage figures: 1.6 KB, 59.5 KB, 512 B.""" - for factor, suffix in _UNITS: - if size >= factor: - return f"{size / factor:.1f}".removesuffix(".0") + f" {suffix}" - return f"{size} B" - - -def _floor_size(size: int) -> str: - """A free size: exact when some unit shows it with one decimal (1920 KB, 4.5 KB), else - rounded down to three significant digits with ≈, so it never overstates (≈1.99 GB).""" - for factor, suffix in _UNITS: - value = size / factor - if value >= 1 and round(value, 1) == value: - return f"{value:.1f}".removesuffix(".0") + f" {suffix}" - for factor, suffix in _UNITS: - if size >= factor: - value = size / factor - decimals = 0 if value >= 100 else 1 if value >= 10 else 2 - floored = math.floor(value * 10**decimals) / 10**decimals - text = f"{floored:.{decimals}f}" - if "." in text: - text = text.rstrip("0").rstrip(".") - exact = floored * factor == size - return f"{'' if exact else '≈'}{text} {suffix}" - return f"{size} B" - - -def _name_list(names: list[str]) -> str: - """Section names for a note, cut short after _NOTE_NAMES.""" - shown = ", ".join(names[:_NOTE_NAMES]) - extra = len(names) - _NOTE_NAMES - return f"{shown}, +{extra} more" if extra > 0 else shown - - -class _Importer: - """Places the sections of one ELF into buckets, collecting diagnostics as it goes.""" - - def __init__(self, elf: Elf, memories: list[MemorySpec], min_size: int) -> None: - """Prepare one bucket per memory region and one for the top level.""" - self.elf = elf - self.memories = memories - self.min_size = min_size - self.ids = _Ids([memory.id for memory in memories]) - self.buckets = [_Bucket(memory) for memory in sorted(memories, key=lambda m: m.start)] - self.top = _Bucket(None) - self.used: dict[str, int] = {} # bytes taken in each memory region, before folding - self.diagnostics: list[Diagnostic] = [] - - def hex(self, value: int) -> str: - """An address at this ELF's width, the way diagnostics print it.""" - return diagnostic_address(value, self.elf.bits) - - def bucket_for(self, item: _Item) -> _Bucket: - """The memory region holding `item`, or the top level. An item that crosses a region - edge means the `--region` specs are wrong.""" - for bucket in self.buckets: - memory = bucket.memory - assert memory is not None - if memory.start <= item.start and item.end <= memory.end: - return bucket - if item.start <= memory.end and memory.start <= item.end: - raise ImportProblem( - f"{item.name} ({self.hex(item.start)} - {self.hex(item.end)}) crosses the " - f"edge of --region {memory.id} ({self.hex(memory.start)} - " - f"{self.hex(memory.end)}); check its start and size" - ) - if self.memories: - self.diagnostics.append( - Diagnostic( - "W110", - f"{item.name} ({self.hex(item.start)} - {self.hex(item.end)}) is outside " - "every --region, so it is placed at the top level", - hint="add a --region that covers it, or check the start and size of yours", - subject=item.id, - ) - ) - return self.top - - def place(self) -> None: - """Put every wanted section in a bucket where it runs, and each load image where it is - stored.""" - wanted = [section for section in self.elf.sections if _wanted(section)] - for section in wanted: - item = _Item(self.ids.make(section.name), section.name, section.address, section.size) - self.bucket_for(item).items.append(item) - for image in self.load_images(wanted): - self.bucket_for(image).items.append(image) - - def load_images(self, sections: list[Section]) -> list[_Item]: - """One block per loadable segment that is stored somewhere other than where it runs, - such as the initial values of .data. Startup code copies each in one piece.""" - images = [] - for segment in self.elf.segments: - if segment.kind != PT_LOAD or segment.file_size == 0: - continue - if segment.physical == segment.virtual: - continue - members = [ - section - for section in sections - if not section.nobits - and segment.offset <= section.offset < segment.offset + segment.file_size - ] - if not members: - continue - target = format_address(segment.virtual, self.elf.bits // 4) - names = [section.name for section in members] - if len(members) == 1: # with several banks, a bare address leaves the reader to look - bank = self.memory_at(segment.virtual) - where = f"{bank.name} at {target}" if bank else target - name, note = f"{names[0]} (load)", f"copied to {where}" - else: - name, note = self.image_name(segment.virtual, members), f"copied to {target}: " - note += _name_list(names) - images.append( - _Item(self.ids.make(name), name, segment.physical, segment.file_size, note=note) - ) - return images - - def memory_at(self, address: int) -> MemorySpec | None: - """The `--region` that holds `address`, if any.""" - return next((m for m in self.memories if m.start <= address <= m.end), None) - - def image_name(self, address: int, members: list[Section]) -> str: - """A name for a load image of several sections, after where it is copied to and what - it holds: RAM init data, or DTIM code and data when it includes code.""" - what = "code and data" if any(s.flags & _SHF_EXECINSTR for s in members) else "init data" - target = self.memory_at(address) - return f"{target.name} {what}" if target else what.capitalize() - - def drop_overlaps(self, bucket: _Bucket) -> None: - """Sort a bucket and leave out any item that overlaps the one before it.""" - kept: list[_Item] = [] - for item in sorted(bucket.items, key=lambda item: (item.start, -item.size)): - if kept and item.start <= kept[-1].end: - self.diagnostics.append( - Diagnostic( - "W111", - f"{item.name} overlaps {kept[-1].name}, so it is left out", - hint="overlays and NOLOAD sections can share addresses; add it by hand", - subject=item.id, - ) - ) - continue - kept.append(item) - bucket.items = kept - - def fold(self, bucket: _Bucket) -> None: - """Group each run of two or more consecutive sections under `min_size` into one block - that holds them as children. A gap of `min_size` or more ends a run, as it gets a Free - block of its own. A region of FOLD_OVER blocks or fewer draws every section in place.""" - if len(bucket.items) <= FOLD_OVER: - return - folded: list[_Item] = [] - run: list[_Item] = [] - - def flush() -> None: - """Move the pending run to `folded`, grouped when it holds more than one item.""" - if len(run) == 1: - folded.append(run[0]) - elif run: - start, end = run[0].start, run[-1].end - group = _Item( - self.ids.make("small_sections"), - f"{len(run)} small sections", - start, - end - start + 1, - note=_name_list([item.name for item in run]), - children=list(run), - ) - folded.append(group) - run.clear() - - for item in bucket.items: - if item.size < self.min_size: - if run and item.start - run[-1].end - 1 >= self.min_size: - flush() - run.append(item) - else: - flush() - folded.append(item) - flush() - bucket.items = folded - - def expand_groups(self) -> None: - """Give every group a detail column, one level deep. When the memory regions are - themselves zoomed out of a whole address space column, which happens when sections lie - outside them, their groups stay one block with a note, as that would be a second level.""" - zoomed_regions = bool(self.top.items) or not self.buckets - buckets = [self.top] if zoomed_regions else self.buckets - for bucket in buckets: - for item in bucket.items: - if item.children: - item.expanded = True - item.note = None # the detail column lists the sections - - def add_free(self, bucket: _Bucket) -> None: - """Add a Free block for each gap of at least `min_size` bytes in a memory region. - Smaller gaps stay unfilled, which draws them as an edge with both addresses.""" - memory = bucket.memory - assert memory is not None - threshold = max(self.min_size, 1) - items: list[_Item] = [] - cursor = memory.start - for item in [*bucket.items, None]: - limit = item.start if item is not None else memory.end + 1 - if limit - cursor >= threshold: - size = limit - cursor - name = f"Free\n{_floor_size(size)}" - items.append(_Item(self.ids.make(f"{memory.id}_free"), name, cursor, size, free=True)) - if item is not None: - items.append(item) - cursor = item.end + 1 - bucket.items = items - - def run(self) -> list[_Bucket]: - """Place, clean up and complete every bucket. Returns the memory region buckets.""" - self.place() - for bucket in [*self.buckets, self.top]: - self.drop_overlaps(bucket) - self.used = { - bucket.memory.id: sum(item.size for item in bucket.items) - for bucket in self.buckets - if bucket.memory is not None - } - if self.min_size > 0: - for bucket in [*self.buckets, self.top]: - self.fold(bucket) - self.expand_groups() - for bucket in self.buckets: - self.add_free(bucket) - return self.buckets - - -def _quote(text: str) -> str: - """A YAML double-quoted scalar. JSON strings are valid YAML.""" - return json.dumps(text, ensure_ascii=False) - - -def _comment_safe(text: str) -> str: - """`text` for a YAML comment: anything that could end the line becomes ?.""" - return "".join(char if char.isprintable() else "?" for char in text) - - -def _size_literal(size: int) -> str: - """A size in the shortest exact form: 64K, 1M, or plain bytes.""" - for factor, suffix in ((1024**3, "G"), (1024**2, "M"), (1024, "K")): - if size % factor == 0: - return f"{size // factor}{suffix}" - return str(size) - - -def _address_literal(value: int, bits: int) -> str: - """An address as zero-padded hex, such as 0x08000000.""" - return f"0x{value:0{bits // 4}X}" - - -def _item_fields(item: _Item, bits: int) -> list[str]: - """The keys of one region, as `key: value` strings.""" - fields = [f"id: {_quote(item.id)}", f"name: {_quote(item.name)}"] - if item.free: - fields.append("kind: reserved") - fields += [f"start: {_address_literal(item.start, bits)}", f"size: {_size_literal(item.size)}"] - if item.note: - fields.append(f"note: {_quote(item.note)}") - return fields - - -def _item_lines(item: _Item, bits: int, indent: str) -> list[str]: - """One region as list entry lines: a flow mapping on one line, or a block mapping with its - children one per line.""" - fields = _item_fields(item, bits) - if not item.children: - return [f"{indent}- {{ " + ", ".join(fields) + " }"] - lines = [f"{indent}- {fields[0]}"] - lines += [f"{indent} {field}" for field in fields[1:]] - lines.append(f"{indent} children:") - for child in item.children: - lines += _item_lines(child, bits, indent + " ") - return lines - - -def _usage_title(memory: MemorySpec, used: int) -> str: - """A column title: the region name, then how much of it the sections use.""" - percent = round(100 * used / memory.size) - if used < memory.size: - percent = min(percent, 99) # 99.6% is not full - shown = "<1" if percent == 0 and used else str(percent) - return f"{memory.name}\n{_approx_size(used)} of {_approx_size(memory.size)} ({shown}%)" - - -def _expansions(buckets: list[_Bucket], top: _Bucket) -> list[tuple[str, str]]: - """Region ids with a detail column of their own, with their titles.""" - found = [] - for bucket in [*buckets, top]: - for item in bucket.items: - if item.expanded: - where = bucket.memory.name if bucket.memory else "memory" - found.append((item.id, f"{item.name} in {where}")) - return found - - -def _write_yaml( - source: str, bits: int, buckets: list[_Bucket], top: _Bucket, used: dict[str, int] -) -> str: - """The config text, laid out like the examples: one line per section.""" - lines = [ - f"# Generated by `memmap from-elf` from {_comment_safe(source)}. Section ranges come from", - "# the ELF and memory regions from --region. Rename, merge or annotate blocks freely.", - "version: 1", - "", - "figure:", - f" title: {_quote(f'{source} memory map')}", - "", - "memory:", - ] - if bits == 64: - lines.append(" address_bits: 64") - lines += [ - " fill_gaps: none # gaps under --min-size show as a split edge", - " regions:", - ] - entries: list[tuple[int, list[str]]] = [] - for bucket in buckets: - memory = bucket.memory - assert memory is not None - block = [ - f" - id: {_quote(memory.id)}", - f" name: {_quote(memory.name)}", - f" start: {_address_literal(memory.start, bits)}", - f" size: {_size_literal(memory.size)}", - ] - if bucket.items: - block.append(" children:") - for item in bucket.items: - block += _item_lines(item, bits, " ") - entries.append((memory.start, block)) - entries += [(item.start, _item_lines(item, bits, " ")) for item in top.items] - for _, block in sorted(entries, key=lambda entry: entry[0]): - lines += block - - lines += [ - "", - "view:", - " defaults:", - " labels: { size: true }", - " columns:", - ] - expand: list[str] = [] - if buckets and not top.items: - for bucket in buckets: - memory = bucket.memory - assert memory is not None - title = _quote(_usage_title(memory, used[memory.id])) - lines.append( - f" - {{ id: {_quote(memory.id + '_col')}, root: {_quote(memory.id)}, title: {title}, " - f"width: {COLUMN_WIDTH} }}" - ) - else: # no regions, or sections outside them: show the whole address space - lines.append(f' - {{ id: all, root: "/", width: {COLUMN_WIDTH} }}') - for bucket in buckets: - memory = bucket.memory - assert memory is not None - title = _quote(_usage_title(memory, used[memory.id])) - expand.append(f" - {{ region: {_quote(memory.id)}, title: {title} }}") - for region, title in _expansions(buckets, top): # a column of its own, for its width - root, column = _quote(region), _quote(f"{region}_col") - lines.append(f" - {{ id: {column}, root: {root}, width: {COLUMN_WIDTH} }}") - expand.append(f" - {{ region: {root}, to: {column}, title: {_quote(title)} }}") - if expand: - lines += [" expand:", *expand] - lines += [ - "", - "style:", - " reserved: { show_label: true } # names the Free blocks", - "", - ] - return "\n".join(lines) - - -def import_elf( - elf: Elf, source: str, memories: list[MemorySpec], min_size: int = 64 -) -> Imported: - """Build a config from an ELF. `source` names the ELF in the header comment and title. - Raises ImportProblem when the sections and the memory regions do not fit together.""" - _check_memories(memories) - if not elf.sections: - raise ImportProblem(f"{source} has no section headers (stripped?), so nothing to draw") - importer = _Importer(elf, memories, min_size) - buckets = importer.run() - if not any(bucket.items for bucket in [*buckets, importer.top]): - raise ImportProblem(f"{source} has no allocated sections to draw") - text = _write_yaml(source, elf.bits, buckets, importer.top, importer.used) - return Imported(text, importer.diagnostics) diff --git a/memmap/from_elf/__init__.py b/memmap/from_elf/__init__.py new file mode 100644 index 0000000..6416d9a --- /dev/null +++ b/memmap/from_elf/__init__.py @@ -0,0 +1,30 @@ +"""`memmap from-elf`: write a config from a linked firmware image. + +- `elf` reads the section and program headers of an ELF file. +- `regions` holds the memory regions sections are placed in, from `--region` specs. +- `linker` reads memory regions from a linker map instead, or as well. +- `importer` places the sections in the regions, and `writer` writes the YAML. +""" + +from .elf import Elf, ElfError, read_elf +from .importer import FOLD_OVER, Imported, import_elf +from .linker import LinkerFile, LinkerRegion, Resolved, read_map, resolve_memories +from .regions import ImportProblem, LeftOut, MemorySpec, parse_region_spec + +__all__ = [ + "FOLD_OVER", + "Elf", + "ElfError", + "ImportProblem", + "Imported", + "LeftOut", + "LinkerFile", + "LinkerRegion", + "MemorySpec", + "Resolved", + "import_elf", + "parse_region_spec", + "read_elf", + "read_map", + "resolve_memories", +] diff --git a/memmap/from_elf/blocks.py b/memmap/from_elf/blocks.py new file mode 100644 index 0000000..ae7020d --- /dev/null +++ b/memmap/from_elf/blocks.py @@ -0,0 +1,43 @@ +"""The blocks an import writes, grouped by the memory region that holds them.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .regions import MemorySpec + + +@dataclass +class Item: + """A block to write: a section, a load image, a folded group of small sections, or free + space. A group holds its sections as `children`; `expanded` gives it a detail column.""" + + id: str + name: str + start: int + size: int + note: str | None = None + free: bool = False + children: list[Item] = field(default_factory=list) + expanded: bool = False + + @property + def end(self) -> int: + """Last address, inclusive.""" + return self.start + self.size - 1 + + +@dataclass +class Bucket: + """The items placed in one memory region, or at the top level when `memory` is None. + `used` counts the bytes its sections take, before folding and free space.""" + + memory: MemorySpec | None + items: list[Item] = field(default_factory=list) + used: int = 0 + + @property + def drawn(self) -> bool: + """Whether the region gets a column: always for `--region`, for a region read from a + linker file only when it holds a section.""" + return self.memory is not None and (bool(self.used) or not self.memory.optional) diff --git a/memmap/elf.py b/memmap/from_elf/elf.py similarity index 100% rename from memmap/elf.py rename to memmap/from_elf/elf.py diff --git a/memmap/from_elf/importer.py b/memmap/from_elf/importer.py new file mode 100644 index 0000000..1475618 --- /dev/null +++ b/memmap/from_elf/importer.py @@ -0,0 +1,317 @@ +"""Turn an ELF file into a memmap config: its allocated sections, placed in memory regions. + +The ELF says where each section runs and where its bytes are loaded from, but not how big flash +and RAM are; those come from `--region` specs or a linker map. The result is ordinary YAML for +the user to edit. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ..diagnostics import Diagnostic +from ..units import diagnostic_address, format_address +from .blocks import Bucket, Item +from .elf import PT_LOAD, SHF_TLS, Elf, Section +from .regions import ImportProblem, LeftOut, MemorySpec, check_memories, unique_id +from .sizes import address_literal, floor_size +from .writer import write_yaml + +_NOTE_NAMES = 6 # section names a note lists before "+N more" +FOLD_OVER = 8 # a region with more blocks than this folds its runs of small sections +_SHF_EXECINSTR = 0x4 + + +@dataclass +class Imported: + """The generated YAML and what the import had to say about the ELF.""" + + text: str + diagnostics: list[Diagnostic] + + +class _Ids: + """Hands out unique region ids derived from section names.""" + + def __init__(self, taken: list[str]) -> None: + """Start with the ids that are already in use.""" + self.taken = set(taken) + + def make(self, text: str) -> str: + """A fresh id from `text`: lowercase, non-alphanumerics as _, suffixed when taken.""" + return unique_id(text, self.taken, "section") + + +class _Crossing(Exception): + """An item runs across the edge of a memory region, so the region is wrong.""" + + def __init__(self, item: Item, memory: MemorySpec) -> None: + """Keep the item and the region it crosses.""" + super().__init__(f"{item.name} crosses {memory.label}") + self.item = item + self.memory = memory + + +def _wanted(section: Section) -> bool: + """Whether a section takes space on the target. Thread-local .tbss only has a template.""" + if not section.allocated or section.size == 0: + return False + return not (section.nobits and section.flags & SHF_TLS) + + +def _name_list(names: list[str]) -> str: + """Section names for a note, cut short after _NOTE_NAMES.""" + shown = ", ".join(names[:_NOTE_NAMES]) + extra = len(names) - _NOTE_NAMES + return f"{shown}, +{extra} more" if extra > 0 else shown + + +class _Importer: + """Places the sections of one ELF into buckets, collecting diagnostics as it goes.""" + + def __init__(self, elf: Elf, memories: list[MemorySpec], min_size: int) -> None: + """Prepare one bucket per memory region and one for the top level.""" + self.elf = elf + self.memories = memories + self.min_size = min_size + self.ids = _Ids([memory.id for memory in memories]) + self.buckets = [Bucket(memory) for memory in sorted(memories, key=lambda m: m.start)] + self.top = Bucket(None) + self.diagnostics: list[Diagnostic] = [] + + def hex(self, value: int) -> str: + """An address at this ELF's width, the way diagnostics print it.""" + return diagnostic_address(value, self.elf.bits) + + def bucket_for(self, item: Item) -> Bucket: + """The memory region holding `item`, or the top level. Raises _Crossing for an item + that crosses a region edge, as the region is then wrong.""" + for bucket in self.buckets: + memory = bucket.memory + assert memory is not None + if memory.start <= item.start and item.end <= memory.end: + return bucket + if item.start <= memory.end and memory.start <= item.end: + raise _Crossing(item, memory) + if self.memories: + from_map = all(memory.source for memory in self.memories) + self.diagnostics.append( + Diagnostic( + "W110", + f"{item.name} ({self.hex(item.start)} - {self.hex(item.end)}) is outside " + "every memory region, so it is placed at the top level", + hint="check that the map belongs to this ELF, or add a --region that " + "covers it" + if from_map + else "add a --region that covers it, or check the start and size of yours", + subject=item.id, + ) + ) + return self.top + + def place(self) -> None: + """Put every wanted section in a bucket where it runs, and each load image where it is + stored.""" + wanted = [section for section in self.elf.sections if _wanted(section)] + for section in wanted: + item = Item(self.ids.make(section.name), section.name, section.address, section.size) + self.bucket_for(item).items.append(item) + for image in self.load_images(wanted): + self.bucket_for(image).items.append(image) + + def load_images(self, sections: list[Section]) -> list[Item]: + """One block per loadable segment that is stored somewhere other than where it runs, + such as the initial values of .data. Startup code copies each in one piece.""" + images = [] + for segment in self.elf.segments: + if segment.kind != PT_LOAD or segment.file_size == 0: + continue + if segment.physical == segment.virtual: + continue + members = [ + section + for section in sections + if not section.nobits + and segment.offset <= section.offset < segment.offset + segment.file_size + ] + if not members: + continue + target = format_address(segment.virtual, self.elf.bits // 4) + names = [section.name for section in members] + if len(members) == 1: # with several banks, a bare address leaves the reader to look + bank = self.memory_at(segment.virtual) + where = f"{bank.name} at {target}" if bank else target + name, note = f"{names[0]} (load)", f"copied to {where}" + else: + name, note = self.image_name(segment.virtual, members), f"copied to {target}: " + note += _name_list(names) + images.append( + Item(self.ids.make(name), name, segment.physical, segment.file_size, note=note) + ) + return images + + def memory_at(self, address: int) -> MemorySpec | None: + """The memory region that holds `address`, if any.""" + return next((m for m in self.memories if m.start <= address <= m.end), None) + + def image_name(self, address: int, members: list[Section]) -> str: + """A name for a load image of several sections, after where it is copied to and what + it holds: RAM init data, or DTIM code and data when it includes code.""" + what = "code and data" if any(s.flags & _SHF_EXECINSTR for s in members) else "init data" + target = self.memory_at(address) + return f"{target.name} {what}" if target else what.capitalize() + + def drop_overlaps(self, bucket: Bucket) -> None: + """Sort a bucket and leave out any item that overlaps the one before it.""" + kept: list[Item] = [] + for item in sorted(bucket.items, key=lambda item: (item.start, -item.size)): + if kept and item.start <= kept[-1].end: + self.diagnostics.append( + Diagnostic( + "W111", + f"{item.name} overlaps {kept[-1].name}, so it is left out", + hint="overlays and NOLOAD sections can share addresses; add it by hand", + subject=item.id, + ) + ) + continue + kept.append(item) + bucket.items = kept + + def fold(self, bucket: Bucket) -> None: + """Group each run of two or more consecutive sections under `min_size` into one block + that holds them as children. A gap of `min_size` or more ends a run, as it gets a Free + block of its own. A region of FOLD_OVER blocks or fewer draws every section in place.""" + if len(bucket.items) <= FOLD_OVER: + return + folded: list[Item] = [] + run: list[Item] = [] + + def flush() -> None: + """Move the pending run to `folded`, grouped when it holds more than one item.""" + if len(run) == 1: + folded.append(run[0]) + elif run: + start, end = run[0].start, run[-1].end + group = Item( + self.ids.make("small_sections"), + f"{len(run)} small sections", + start, + end - start + 1, + note=_name_list([item.name for item in run]), + children=list(run), + ) + folded.append(group) + run.clear() + + for item in bucket.items: + if item.size < self.min_size: + if run and item.start - run[-1].end - 1 >= self.min_size: + flush() + run.append(item) + else: + flush() + folded.append(item) + flush() + bucket.items = folded + + def expand_groups(self) -> None: + """Give every group a detail column, one level deep. When the memory regions are + themselves zoomed out of a whole address space column, which happens when sections lie + outside them, their groups stay one block with a note, as that would be a second level.""" + zoomed_regions = bool(self.top.items) or not any(bucket.drawn for bucket in self.buckets) + buckets = [self.top] if zoomed_regions else self.buckets + for bucket in buckets: + for item in bucket.items: + if item.children: + item.expanded = True + item.note = None # the detail column lists the sections + + def add_free(self, bucket: Bucket) -> None: + """Add a Free block for each gap of at least `min_size` bytes in a memory region. + Smaller gaps stay unfilled, which draws them as an edge with both addresses.""" + memory = bucket.memory + assert memory is not None + threshold = max(self.min_size, 1) + items: list[Item] = [] + cursor = memory.start + for item in [*bucket.items, None]: + limit = item.start if item is not None else memory.end + 1 + if limit - cursor >= threshold: + size = limit - cursor + name = f"Free\n{floor_size(size)}" + items.append(Item(self.ids.make(f"{memory.id}_free"), name, cursor, size, free=True)) + if item is not None: + items.append(item) + cursor = item.end + 1 + bucket.items = items + + def run(self) -> list[Bucket]: + """Place, clean up and complete every bucket. Returns the memory region buckets.""" + self.place() + for bucket in [*self.buckets, self.top]: + self.drop_overlaps(bucket) + for bucket in self.buckets: + bucket.used = sum(item.size for item in bucket.items) + if self.min_size > 0: + for bucket in [*self.buckets, self.top]: + self.fold(bucket) + self.expand_groups() + for bucket in self.buckets: + self.add_free(bucket) + return self.buckets + + +def _regions_from(memories: list[MemorySpec]) -> str: + """Where the memory regions came from, for the header: a map, --region, or both.""" + sources = dict.fromkeys(memory.source or "--region" for memory in memories) + return " and ".join(sorted(sources, key=lambda source: source == "--region")) or "--region" + + +def _without_crossed(crossing: _Crossing, bits: int) -> LeftOut: + """The map region that `crossing` runs across, left out. Raises ImportProblem when it is + a --region instead, as only the user can say which of its edges is wrong.""" + item, memory = crossing.item, crossing.memory + if memory.source is None: + raise ImportProblem( + f"{item.name} ({diagnostic_address(item.start, bits)} - " + f"{diagnostic_address(item.end, bits)}) crosses the edge of {memory.label} " + f"({diagnostic_address(memory.start, bits)} - {diagnostic_address(memory.end, bits)})", + hint="check its start and size", + ) + span = f"{address_literal(item.start, bits)} - {address_literal(item.end, bits)}" + start = address_literal(memory.start, bits) if memory.start <= item.start else "START" + hint = f"give it with --region {memory.id}={start}:SIZE, large enough to hold {item.name}" + reason = f"{item.name} ({span}) runs across its edge" + return LeftOut(memory.id, memory.name, memory.start, memory.size, memory.source, reason, hint) + + +def import_elf( + elf: Elf, + source: str, + memories: list[MemorySpec], + min_size: int = 64, + notes: tuple[str, ...] = (), +) -> Imported: + """Build a config from an ELF. `source` names the ELF in the header comment and title, and + `notes` are comment lines for the header. A map region that a section runs across is left + out with W112, and the sections placed again. Raises ImportProblem when the sections and + the memory regions do not fit together.""" + check_memories(memories) + if not elf.sections: + raise ImportProblem(f"{source} has no section headers (stripped?), so nothing to draw") + left: list[LeftOut] = [] + while True: + importer = _Importer(elf, memories, min_size) + try: + buckets = importer.run() + break + except _Crossing as crossing: + left.append(_without_crossed(crossing, elf.bits)) + memories = [memory for memory in memories if memory is not crossing.memory] + if not any(bucket.items for bucket in [*buckets, importer.top]): + raise ImportProblem(f"{source} has no allocated sections to draw") + header = (*notes, *(entry.note(elf.bits) for entry in left)) + text = write_yaml(source, elf.bits, buckets, importer.top, _regions_from(memories), header) + warnings = [warning for entry in left if (warning := entry.warning(elf.bits))] + return Imported(text, warnings + importer.diagnostics) diff --git a/memmap/from_elf/linker/__init__.py b/memmap/from_elf/linker/__init__.py new file mode 100644 index 0000000..1f5b846 --- /dev/null +++ b/memmap/from_elf/linker/__init__.py @@ -0,0 +1,17 @@ +"""Memory regions from the map file a linker writes. + +| Toolchain | Map | Module | +| --- | --- | --- | +| GNU ld | `-Map`: the Memory Configuration table | `gnu_map` | +| Arm armlink (Keil MDK) | `--map`: each execution region's base and Max | `armlink_map` | +| IAR ILINK | `--map`: the ranges of the placement summary | `iar_map` | +| LLVM lld | `-Map`, but it lists no regions: `--region` gives them | `files` tells it apart | + +`files` tells the formats apart, and `resolve` turns the regions into MemorySpecs. +""" + +from .files import read_map +from .model import LinkerFile, LinkerRegion +from .resolve import Resolved, resolve_memories + +__all__ = ["LinkerFile", "LinkerRegion", "Resolved", "read_map", "resolve_memories"] diff --git a/memmap/from_elf/linker/armlink_map.py b/memmap/from_elf/linker/armlink_map.py new file mode 100644 index 0000000..59f74c7 --- /dev/null +++ b/memmap/from_elf/linker/armlink_map.py @@ -0,0 +1,39 @@ +"""Arm linker map files (armlink `--map`, as Keil MDK writes them): the execution regions. + + Load Region LR_IROM1 (Base: 0x08000000, Size: 0x0000158c, Max: 0x00010000, ABSOLUTE) + Execution Region ER_IROM1 (Base: 0x08000000, Size: 0x00001538, Max: 0x00010000, ABSOLUTE) + Execution Region RW_IRAM1 (Exec base: 0x20000000, Load base: 0x08005434, Size: ... + +Execution regions are where code and data run, and `Max` is the size the scatter file gives +them. armlink 5.05 writes `Base:`, and 5.06 `Exec base:` and `Load base:`; no map from the +armlink of Arm Compiler 6 is tested yet. A region placed with --ro-base or --rw-base has no +limit, which the map shows as a Max of all ones. +""" + +from __future__ import annotations + +import re + +from .model import LinkerRegion + +NAME = "armlink map" +_REGION = re.compile( + r"^[ \t]*Execution Region[ \t]+(\S+)[ \t]+\((?:Exec base|Base):[ \t]*(0x[0-9a-fA-F]+)," + r"[^)\n]*?\bMax:[ \t]*(0x[0-9a-fA-F]+)", + re.MULTILINE, +) + + +def matches(text: str) -> bool: + """Whether `text` is an armlink map with a memory map of the image.""" + return "Memory Map of the image" in text and bool(_REGION.search(text)) + + +def read(text: str) -> list[LinkerRegion]: + """Each execution region with its base and maximum size.""" + regions = [] + for match in _REGION.finditer(text): + name, base, limit = match.group(1), int(match.group(2), 16), match.group(3) + unlimited = set(limit[2:].lower()) == {"f"} + regions.append(LinkerRegion(name, base, None if unlimited else int(limit, 16))) + return regions diff --git a/memmap/from_elf/linker/files.py b/memmap/from_elf/linker/files.py new file mode 100644 index 0000000..1cc0eff --- /dev/null +++ b/memmap/from_elf/linker/files.py @@ -0,0 +1,52 @@ +"""Read a linker map, telling the formats apart by their content. + +GNU ld and armlink list each region with its size. IAR lists the ranges it placed content in. +An lld map has no region table at all, so it is read as a map without regions. A linker script +is turned away: memmap reads what the linker worked out, not what it was told. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from ..regions import ImportProblem +from . import armlink_map, gnu_map, iar_map +from .model import LLD_MAP, LinkerFile + +_MAPS = (gnu_map, armlink_map, iar_map) +# lld writes a column header first: VMA LMA Size Align Out In Symbol, or Address without LMA. +_LLD_MAP = re.compile(r"^\s*(?:VMA\s+LMA|Address)\s+Size\s+Align\s+Out\s+In\s+Symbol\b") +_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) +_SCRIPT = re.compile(r"\b(?:MEMORY|SECTIONS)\s*\{") +_MEMORY = re.compile(r"\bMEMORY\s*\{") + + +def _text(path: Path) -> str: + """The file's text with Unix line ends and without a byte order mark. Maps from Windows + tools are not always UTF-8.""" + try: + text = path.read_bytes().decode("utf-8-sig", "replace") + except OSError as exc: + raise ImportProblem(f"cannot read {path}: {exc.strerror or exc}") from None + return text.replace("\r\n", "\n").replace("\r", "\n") + + +def read_map(path: Path) -> LinkerFile: + """The regions of a GNU ld, armlink or IAR map file, or none for an lld map.""" + text = _text(path) + for reader in _MAPS: + if reader.matches(text): + return LinkerFile(path.name, reader.NAME, reader.read(text)) + if _LLD_MAP.match(text): + return LinkerFile(path.name, LLD_MAP) + script = _COMMENT.sub(" ", text) + if _SCRIPT.search(script): + hint = "pass the map file the linker writes, or give each region with --region ID=START:SIZE" + if _MEMORY.search(script): + hint += " from the script's MEMORY block" + raise ImportProblem(f"{path.name} is a linker script, which memmap does not read", hint) + raise ImportProblem( + f"{path.name} is not a map file memmap can read", + hint="pass the map file of a GNU ld, armlink or IAR link", + ) diff --git a/memmap/from_elf/linker/gnu_map.py b/memmap/from_elf/linker/gnu_map.py new file mode 100644 index 0000000..c60d00d --- /dev/null +++ b/memmap/from_elf/linker/gnu_map.py @@ -0,0 +1,46 @@ +"""GNU ld map files (`-Wl,-Map=firmware.map`): the Memory Configuration table. + + Memory Configuration + + Name Origin Length Attributes + FLASH 0x08000000 0x00010000 xr + *default* 0x00000000 0xffffffff + +The values are the MEMORY block of the linker script, already worked out. A 64-bit ld prints +every address at 16 digits, even for a 32-bit target. +""" + +from __future__ import annotations + +import re + +from ..regions import ImportProblem +from .model import LinkerRegion + +NAME = "GNU ld map" +_TABLE = re.compile(r"^Memory Configuration[ \t]*$", re.MULTILINE) +_END = "Linker script and memory map" + + +def matches(text: str) -> bool: + """Whether `text` is a GNU ld map: it has the Memory Configuration table.""" + return bool(_TABLE.search(text)) + + +def read(text: str) -> list[LinkerRegion]: + """The rows of the Memory Configuration table, except the catch-all *default* region.""" + found = _TABLE.search(text) + assert found is not None + regions = [] + for line in text[found.end() :].splitlines(): + if line.startswith(_END): + break + fields = line.split() + if len(fields) < 3 or fields[0] in ("Name", "*default*"): + continue + try: + origin, length = int(fields[1], 16), int(fields[2], 16) + except ValueError: + raise ImportProblem(f"cannot read the memory configuration line {line.strip()!r}") from None + regions.append(LinkerRegion(fields[0], origin, length)) + return regions diff --git a/memmap/from_elf/linker/iar_map.py b/memmap/from_elf/linker/iar_map.py new file mode 100644 index 0000000..a061bd5 --- /dev/null +++ b/memmap/from_elf/linker/iar_map.py @@ -0,0 +1,120 @@ +"""IAR ILINK map files: the address ranges of the placement summary. + + "A0": place at address 0x800'0000 { ro section .intvec }; + "P1": place in [from 0x800'0000 to 0x80f'ffff] { ro }; + "P2": place in [from 0x2000'0000 to 0x2001'bfff] | + [from 0x2001'c000 to 0x2001'ffff] { rw, block CSTACK, block HEAP }; + +Each `place in` statement names the ranges of a region from the .icf file, worked out, but not +the region's name. So overlapping and adjoining ranges are merged into one region, called RAM +when read-write data or a stack goes there, ROM when only read-only content does, and Memory +when the statements do not say, as for named sections only. Newer versions write digit groups +with ', and some older ones write a range as [0x8000-0xffff]. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from ..regions import ImportProblem +from .model import LinkerRegion + +NAME = "IAR map" +_SUMMARY = "*** PLACEMENT SUMMARY" +_NUMBER = r"(0x[0-9a-fA-F']+|[0-9][0-9']*)" +_RANGES = ( + (re.compile(rf"\[\s*from\s+{_NUMBER}\s+to\s+{_NUMBER}\s*\]"), False), + (re.compile(rf"\[\s*from\s+{_NUMBER}\s+size\s+{_NUMBER}\s*\]"), True), + (re.compile(rf"\[\s*{_NUMBER}\s*-\s*{_NUMBER}\s*\]"), False), +) +_PLACE = re.compile(r'"[^"]+":\s*place\s+(?:in|at start of|at end of)\s+([^{]*)\{([^}]*)\}') +_READ_ONLY = re.compile(r"^(ro|readonly)\b") +_READ_WRITE = re.compile(r"^(rw|readwrite|zi|zeroinit)\b|^block\s+\w*(STACK|HEAP)", re.IGNORECASE) +_NAMES = {"rw": "RAM", "ro": "ROM", "": "Memory"} + + +@dataclass +class _Range: + """An address range from one statement, and what goes there: rw for read-write data or a + stack, ro for read-only content only, or "" when the statement does not say.""" + + start: int + end: int + kind: str + + +def _kind(items: list[str]) -> str: + """What a statement places: rw, ro or "".""" + if any(_READ_WRITE.match(item) for item in items): + return "rw" + if items and all(_READ_ONLY.match(item) for item in items): + return "ro" + return "" + + +def _merged_kind(first: str, second: str) -> str | None: + """The kind of two ranges together, or None when they hold different kinds of content.""" + if first and second and first != second: + return None + return first or second + + +def matches(text: str) -> bool: + """Whether `text` is an IAR ILINK map with a placement summary.""" + return "IAR ELF Linker" in text[:4096] and _SUMMARY in text + + +def _number(text: str) -> int: + """A number as IAR writes it, with ' between digit groups.""" + digits = text.replace("'", "") + try: + return int(digits, 16) if digits[:2].lower() == "0x" else int(digits) + except ValueError: + raise ImportProblem(f"cannot read the address {text!r} in the placement summary") from None + + +def _ranges(text: str) -> list[_Range]: + """The ranges of every `place in` and `place at start/end of` statement, except any that + ends before it starts, as no linker writes one.""" + found = [] + block = text[text.index(_SUMMARY) + len(_SUMMARY) :] + table = re.search(r"^\s+Section\s+Kind\s+Address", block, re.MULTILINE) + for statement in block[: table.start() if table else len(block)].split(";"): + statement = " ".join(statement.split()) + placed = _PLACE.search(statement) + if not placed: + continue + where, content = placed.groups() + kind = _kind([item.strip() for item in content.split(",") if item.strip()]) + for pattern, sized in _RANGES: + for match in pattern.finditer(where): + start, last = _number(match.group(1)), _number(match.group(2)) + end = start + last - 1 if sized else last + if end >= start: + found.append(_Range(start, end, kind)) + return found + + +def read(text: str) -> list[LinkerRegion]: + """The merged placement ranges as regions named RAM, ROM or Memory, numbered when + repeated. Overlapping ranges always merge, adjoining ones when they hold the same kind.""" + merged: list[_Range] = [] + for item in sorted(_ranges(text), key=lambda item: (item.start, item.end)): + last = merged[-1] if merged else None + kind = _merged_kind(last.kind, item.kind) if last else None + if last is not None and item.start <= last.end: + last.end = max(last.end, item.end) + last.kind = "rw" if kind is None else kind + elif last is not None and item.start == last.end + 1 and kind is not None: + last.end, last.kind = item.end, kind + else: + merged.append(_Range(item.start, item.end, item.kind)) + counts: dict[str, int] = {} + regions = [] + for item in merged: + base = _NAMES[item.kind] + counts[base] = counts.get(base, 0) + 1 + name = base if counts[base] == 1 else f"{base} {counts[base]}" + regions.append(LinkerRegion(name, item.start, item.end - item.start + 1)) + return regions diff --git a/memmap/from_elf/linker/model.py b/memmap/from_elf/linker/model.py new file mode 100644 index 0000000..33332ed --- /dev/null +++ b/memmap/from_elf/linker/model.py @@ -0,0 +1,27 @@ +"""What memmap takes from a linker map: its memory regions.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +LLD_MAP = "lld map" + + +@dataclass(frozen=True) +class LinkerRegion: + """One memory region, named as the map names it. `size` is None when the map sets no + limit, as armlink does for a region placed without a scatter file.""" + + name: str + start: int + size: int | None + + +@dataclass +class LinkerFile: + """The regions read from one map. `kind` names its format, such as "GNU ld map". An lld + map, or a GNU ld map of a script without a MEMORY block, has no regions.""" + + name: str + kind: str + regions: list[LinkerRegion] = field(default_factory=list) diff --git a/memmap/from_elf/linker/resolve.py b/memmap/from_elf/linker/resolve.py new file mode 100644 index 0000000..d8c083e --- /dev/null +++ b/memmap/from_elf/linker/resolve.py @@ -0,0 +1,181 @@ +"""Turn the regions of a linker map into memory regions for the import. + +Each region gets an id from its name, FLASH as flash. A `--region` with the same id replaces +it, and one with a new id is added. A map region that cannot be drawn is left out: one without +a size or of size 0, one that runs past the address space, and one that overlaps a `--region` +or a lower or larger region of the map, as when a script declares one RAM twice, for its +instruction and data bus. + +Each region left out is noted in the config's header. It is also reported as W112, with what to +give instead, unless nothing in it is lost: a region of size 0, one inside the region it +overlaps, or one whose start lies in a drawn region. A region that holds no section is not left +out: it stays in `memory` without a column, like the IDT_LIST region Zephyr declares near the +top of memory for a build step of its own. + +When nothing is left to draw, because the map lists no regions, as an lld map does, or none +that can be drawn, and no `--region` stands in, the import stops and says what to give instead. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace + +from ...diagnostics import Diagnostic +from ..regions import ( + ImportProblem, + LeftOut, + MemorySpec, + check_memories, + describe_region, + unique_id, +) +from ..sizes import address_literal, size_literal +from . import gnu_map, iar_map +from .model import LLD_MAP, LinkerFile, LinkerRegion + + +@dataclass +class Resolved: + """What the import gets from a map: the regions to draw, comment lines for the config's + header, and W112 for each region left out that loses something.""" + + memories: list[MemorySpec] + notes: list[str] = field(default_factory=list) + diagnostics: list[Diagnostic] = field(default_factory=list) + + +def region_id(name: str, taken: set[str]) -> str: + """An id for a region name: lowercase, non-alphanumerics as _, suffixed when taken.""" + return unique_id(name, taken, "region") + + +def _why_left_out(region: LinkerRegion, bits: int) -> str | None: + """Why a region cannot be drawn, or None when it can.""" + if region.size is None: + return "no size" + if region.size == 0: + return "size 0" + if region.start + region.size > 1 << bits: + return f"ends past the {bits}-bit address space" + return None + + +def _empty(linker: LinkerFile) -> str: + """Why a map lists no regions at all.""" + if linker.kind == LLD_MAP: + return f"{linker.name} is an lld map, which lists no memory regions" + if linker.kind == gnu_map.NAME: + return f"{linker.name} lists no memory regions, as its linker script has no MEMORY block" + return f"{linker.name} lists no memory regions" + + +def _spec(rid: str, start: int, bits: int) -> str: + """The --region to give for a region whose start is known but not its size.""" + return f"--region {rid}={address_literal(start, bits)}:SIZE" + + +def _nothing_to_draw(linker: LinkerFile, unusable: list[LeftOut], bits: int) -> ImportProblem: + """The problem of a map that leaves no region to draw, with the --region specs to give + instead as its hint: one per start of a region without a size.""" + if not linker.regions: + hint = "give each region with --region ID=START:SIZE" + if linker.kind == LLD_MAP: + hint += ", copying ORIGIN and LENGTH from the linker script's MEMORY block" + return ImportProblem(_empty(linker), hint) + starts: dict[int, str] = {} + for entry in unusable: + if entry.size is None: + starts.setdefault(entry.start, entry.id) + reasons = "; ".join( + f"{describe_region(entry.name, entry.start, entry.size, bits)}: {entry.reason}" + for entry in unusable + ) + specs = " ".join(_spec(rid, start, bits) for start, rid in starts.items()) + hint = f"give them with --region, such as {specs}" if specs else ( + "give each region with --region ID=START:SIZE" + ) + return ImportProblem(f"{linker.name} has no region memmap can draw ({reasons})", hint) + + +def _overlap_hint(memory: MemorySpec, other: MemorySpec, bits: int) -> str | None: + """What to give for a map region left out as it overlaps `other`, or None when it lies + inside `other`, so nothing in it is lost.""" + if other.start <= memory.start and memory.end <= other.end: + return None + if other.source is None: + return f"to split {memory.name}, give one of the --region specs its id, {memory.id}" + spec = f"{other.id}={address_literal(memory.start, bits)}:{size_literal(memory.size)}" + return f"to draw it instead of {other.name}, give it with --region {spec}:{memory.name}" + + +def _without_overlaps( + memories: list[MemorySpec], source: str, bits: int +) -> tuple[list[MemorySpec], list[LeftOut]]: + """`memories` without each map region that overlaps a --region, or a map region that + starts lower or, at the same start, is larger or listed first; and those left out.""" + kept = [memory for memory in memories if memory.source is None] + left: list[LeftOut] = [] + for memory in sorted((m for m in memories if m.source), key=lambda m: (m.start, -m.size)): + other = next((k for k in kept if memory.start <= k.end and k.start <= memory.end), None) + if other is None: + kept.append(memory) + continue + reason = f"overlaps {other.name if other.source else other.label}" + hint = _overlap_hint(memory, other, bits) + left.append(LeftOut(memory.id, memory.name, memory.start, memory.size, source, reason, hint)) + return [memory for memory in memories if memory in kept], left + + +def _with_hints(unusable: list[LeftOut], drawn: list[MemorySpec], bits: int) -> list[LeftOut]: + """`unusable` with the hint for each W112. A region of size 0 holds nothing, and one whose + start lies in a drawn region is drawn there, so neither gets one; nor does a second region + at the same start, as one --region covers both.""" + warned: set[int] = set() + hinted = [] + for entry in unusable: + covered = any(memory.start <= entry.start <= memory.end for memory in drawn) + if entry.size == 0 or covered or entry.start in warned: + hinted.append(entry) + continue + warned.add(entry.start) + hint = f"give it with {_spec(entry.id, entry.start, bits)}" + if entry.size is not None: # past the address space + hint = f"check that the map belongs to this ELF, or {hint}" + hinted.append(replace(entry, hint=hint)) + return hinted + + +def resolve_memories(linker: LinkerFile, specs: list[MemorySpec], bits: int) -> Resolved: + """The memory regions of `linker`, with `specs` from --region replacing or adding to + them. Raises ImportProblem when no region is left to draw, or when the specs repeat an id + or overlap each other.""" + check_memories(specs) + overrides = {spec.id: spec for spec in specs} + taken: set[str] = set() + memories: list[MemorySpec] = [] + unusable: list[LeftOut] = [] + for region in linker.regions: + rid = region_id(region.name, taken) + if rid in overrides: + memories.append(overrides.pop(rid)) + continue + reason = _why_left_out(region, bits) + if reason is None: + assert region.size is not None + memories.append(MemorySpec(rid, region.start, region.size, region.name, linker.name)) + continue + unusable.append(LeftOut(rid, region.name, region.start, region.size, linker.name, reason)) + memories += overrides.values() + if not memories: + raise _nothing_to_draw(linker, unusable, bits) + memories, overlapping = _without_overlaps(memories, linker.name, bits) + left = _with_hints(unusable, memories, bits) + overlapping + notes = [] if linker.regions else [f"{_empty(linker)}."] + if linker.kind == iar_map.NAME: + notes.append( + f"{linker.name} is an IAR map, which names no regions: these are its placement " + "ranges, called ROM, RAM or Memory by what is placed there." + ) + notes += [entry.note(bits) for entry in left] + warnings = [entry.warning(bits) for entry in left] + return Resolved(memories, notes, [warning for warning in warnings if warning]) diff --git a/memmap/from_elf/regions.py b/memmap/from_elf/regions.py new file mode 100644 index 0000000..3714280 --- /dev/null +++ b/memmap/from_elf/regions.py @@ -0,0 +1,129 @@ +"""Memory regions for an import: what `--region` gives, and how they are checked. + +The ELF says where each section runs, not how big flash and RAM are. Regions come from +`--region` specs, from a linker map (see `linker`), or both. +""" + +from __future__ import annotations + +import itertools +import re +from dataclasses import dataclass + +from ..diagnostics import Diagnostic +from ..units import parse_address, parse_size +from .sizes import address_literal, size_literal + +ID_RE = re.compile(r"^[a-z0-9_]+$") + + +class ImportProblem(ValueError): + """The ELF and the memory regions do not fit together, or a region is malformed. `hint` + says what to give instead, when there is something to say.""" + + def __init__(self, message: str, hint: str | None = None) -> None: + """Keep the message and the hint apart, as --json reports them apart.""" + super().__init__(message) + self.hint = hint + + +def unique_id(text: str, taken: set[str], fallback: str) -> str: + """An id from `text`: lowercase, non-alphanumerics as _, `fallback` when nothing is left, + suffixed _2, _3 and so on when taken. Adds it to `taken`.""" + base = re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") or fallback + candidate, number = base, 2 + while candidate in taken: + candidate, number = f"{base}_{number}", number + 1 + taken.add(candidate) + return candidate + + +def describe_region(name: str, start: int, size: int | None, bits: int) -> str: + """A region for a message: `FLASH (0x08000000, 64K)`, or without the size when it has none.""" + where = address_literal(start, bits) + return f"{name} ({where})" if size is None else f"{name} ({where}, {size_literal(size)})" + + +@dataclass(frozen=True) +class MemorySpec: + """One memory region: an id, a range and a display name. `source` names the map it was + read from, or is None for a `--region` spec. A region from a map is only drawn when it + holds a section.""" + + id: str + start: int + size: int + name: str + source: str | None = None + + @property + def end(self) -> int: + """Last address, inclusive.""" + return self.start + self.size - 1 + + @property + def optional(self) -> bool: + """Whether the region is left without a column when it holds nothing.""" + return self.source is not None + + @property + def label(self) -> str: + """How messages name the region: `--region flash` or `FLASH in firmware.map`.""" + return f"--region {self.id}" if self.source is None else f"{self.name} in {self.source}" + + +@dataclass(frozen=True) +class LeftOut: + """A map region the import does not draw, and why. Each one is noted in the config's + header. One with a `hint` loses something, so it is also reported as W112.""" + + id: str + name: str + start: int + size: int | None + source: str + reason: str + hint: str | None = None + + def note(self, bits: int) -> str: + """The header comment, such as `Left out of fw.map: RAM (0x20000000, 20K): size 0.`""" + described = describe_region(self.name, self.start, self.size, bits) + return f"Left out of {self.source}: {described}: {self.reason}." + + def warning(self, bits: int) -> Diagnostic | None: + """W112 for the region, or None when nothing in it is lost.""" + if self.hint is None: + return None + described = describe_region(self.name, self.start, self.size, bits) + message = f"{described} in {self.source} is left out: {self.reason}" + return Diagnostic("W112", message, hint=self.hint, subject=self.id) + + +def parse_region_spec(text: str) -> MemorySpec: + """Parse `id=start:size[:name]`, such as `flash=0x08000000:64K:Internal flash`.""" + region_id, separator, rest = text.partition("=") + parts = rest.split(":", 2) + if not separator or len(parts) < 2: + raise ImportProblem(f"bad --region {text!r}, expected id=start:size[:name]") + region_id = region_id.strip() + if not ID_RE.match(region_id): + raise ImportProblem(f"bad --region id {region_id!r}, use lowercase letters, digits and _") + try: + start, size = parse_address(parts[0]), parse_size(parts[1]) + except ValueError as exc: + raise ImportProblem(f"bad --region {text!r}: {exc}") from None + name = parts[2].strip() if len(parts) == 3 and parts[2].strip() else region_id.upper() + return MemorySpec(region_id, start, size, name) + + +def check_memories(memories: list[MemorySpec]) -> None: + """Reject duplicate ids and overlapping regions.""" + seen: set[str] = set() + for memory in memories: + if memory.id in seen: + raise ImportProblem(f"--region {memory.id} is given twice") + seen.add(memory.id) + ordered = sorted(memories, key=lambda memory: memory.start) + for low, high in itertools.pairwise(ordered): + if high.start <= low.end: + raise ImportProblem(f"{low.label} and {high.label} overlap") diff --git a/memmap/from_elf/sizes.py b/memmap/from_elf/sizes.py new file mode 100644 index 0000000..d0c255c --- /dev/null +++ b/memmap/from_elf/sizes.py @@ -0,0 +1,48 @@ +"""Sizes and addresses as an import writes them: in names, titles and YAML literals.""" + +from __future__ import annotations + +import math + +_UNITS = ((1024**3, "GB"), (1024**2, "MB"), (1024, "KB")) + + +def approx_size(size: int) -> str: + """A size rounded to one decimal, for usage figures: 1.6 KB, 59.5 KB, 512 B.""" + for factor, suffix in _UNITS: + if size >= factor: + return f"{size / factor:.1f}".removesuffix(".0") + f" {suffix}" + return f"{size} B" + + +def floor_size(size: int) -> str: + """A free size: exact when some unit shows it with one decimal (1920 KB, 4.5 KB), else + rounded down to three significant digits with ≈, so it never overstates (≈1.99 GB).""" + for factor, suffix in _UNITS: + value = size / factor + if value >= 1 and round(value, 1) == value: + return f"{value:.1f}".removesuffix(".0") + f" {suffix}" + for factor, suffix in _UNITS: + if size >= factor: + value = size / factor + decimals = 0 if value >= 100 else 1 if value >= 10 else 2 + floored = math.floor(value * 10**decimals) / 10**decimals + text = f"{floored:.{decimals}f}" + if "." in text: + text = text.rstrip("0").rstrip(".") + exact = floored * factor == size + return f"{'' if exact else '≈'}{text} {suffix}" + return f"{size} B" + + +def size_literal(size: int) -> str: + """A size in the shortest exact form: 64K, 1M, or plain bytes.""" + for factor, suffix in ((1024**3, "G"), (1024**2, "M"), (1024, "K")): + if size and size % factor == 0: + return f"{size // factor}{suffix}" + return str(size) + + +def address_literal(value: int, bits: int) -> str: + """An address as zero-padded hex, such as 0x08000000.""" + return f"0x{value:0{bits // 4}X}" diff --git a/memmap/from_elf/writer.py b/memmap/from_elf/writer.py new file mode 100644 index 0000000..ab110c1 --- /dev/null +++ b/memmap/from_elf/writer.py @@ -0,0 +1,167 @@ +"""Write an import as config text, laid out like the examples: one line per section.""" + +from __future__ import annotations + +import json + +from .blocks import Bucket, Item +from .sizes import address_literal, approx_size, size_literal + +COLUMN_WIDTH = 150 # px, room for section names such as .ARM.exidx and the usage title + + +def _quote(text: str) -> str: + """A YAML double-quoted scalar. JSON strings are valid YAML.""" + return json.dumps(text, ensure_ascii=False) + + +def comment_safe(text: str) -> str: + """`text` for a YAML comment: anything that could end the line becomes ?.""" + return "".join(char if char.isprintable() else "?" for char in text) + + +def _item_fields(item: Item, bits: int) -> list[str]: + """The keys of one region, as `key: value` strings.""" + fields = [f"id: {_quote(item.id)}", f"name: {_quote(item.name)}"] + if item.free: + fields.append("kind: reserved") + fields += [f"start: {address_literal(item.start, bits)}", f"size: {size_literal(item.size)}"] + if item.note: + fields.append(f"note: {_quote(item.note)}") + return fields + + +def _item_lines(item: Item, bits: int, indent: str) -> list[str]: + """One region as list entry lines: a flow mapping on one line, or a block mapping with its + children one per line.""" + fields = _item_fields(item, bits) + if not item.children: + return [f"{indent}- {{ " + ", ".join(fields) + " }"] + lines = [f"{indent}- {fields[0]}"] + lines += [f"{indent} {field}" for field in fields[1:]] + lines.append(f"{indent} children:") + for child in item.children: + lines += _item_lines(child, bits, indent + " ") + return lines + + +def _usage_title(bucket: Bucket) -> str: + """A column title: the region name, then how much of it the sections use.""" + memory = bucket.memory + assert memory is not None + used = bucket.used + percent = round(100 * used / memory.size) + if used < memory.size: + percent = min(percent, 99) # 99.6% is not full + shown = "<1" if percent == 0 and used else str(percent) + return f"{memory.name}\n{approx_size(used)} of {approx_size(memory.size)} ({shown}%)" + + +def _expansions(buckets: list[Bucket], top: Bucket) -> list[tuple[str, str]]: + """Region ids with a detail column of their own, with their titles.""" + found = [] + for bucket in [*buckets, top]: + for item in bucket.items: + if item.expanded: + where = bucket.memory.name if bucket.memory else "memory" + found.append((item.id, f"{item.name} in {where}")) + return found + + +def _memory_lines(bits: int, buckets: list[Bucket], top: Bucket) -> list[str]: + """The `memory` section: each region with its blocks, and the blocks outside every region, + in address order.""" + lines = ["memory:"] + if bits == 64: + lines.append(" address_bits: 64") + lines += [ + " fill_gaps: none # gaps under --min-size show as a split edge", + " regions:", + ] + entries: list[tuple[int, list[str]]] = [] + for bucket in buckets: + memory = bucket.memory + assert memory is not None + block = [ + f" - id: {_quote(memory.id)}", + f" name: {_quote(memory.name)}", + f" start: {address_literal(memory.start, bits)}", + f" size: {size_literal(memory.size)}", + ] + if bucket.items: + block.append(" children:") + for item in bucket.items: + block += _item_lines(item, bits, " ") + entries.append((memory.start, block)) + entries += [(item.start, _item_lines(item, bits, " ")) for item in top.items] + for _, block in sorted(entries, key=lambda entry: entry[0]): + lines += block + return lines + + +def _view_lines(buckets: list[Bucket], top: Bucket) -> list[str]: + """The `view` section: a column per drawn region, or one for the whole address space with + the regions zoomed out of it, plus a detail column per folded group.""" + lines = ["view:", " defaults:", " labels: { size: true }", " columns:"] + drawn = [bucket for bucket in buckets if bucket.drawn] + expand: list[str] = [] + if drawn and not top.items: + for bucket in drawn: + memory = bucket.memory + assert memory is not None + title = _quote(_usage_title(bucket)) + lines.append( + f" - {{ id: {_quote(memory.id + '_col')}, root: {_quote(memory.id)}, title: {title}, " + f"width: {COLUMN_WIDTH} }}" + ) + else: # no regions, or sections outside them: show the whole address space + lines.append(f' - {{ id: all, root: "/", width: {COLUMN_WIDTH} }}') + for bucket in drawn: + memory = bucket.memory + assert memory is not None + title = _quote(_usage_title(bucket)) + expand.append(f" - {{ region: {_quote(memory.id)}, title: {title} }}") + empty = [bucket.memory for bucket in buckets if not bucket.drawn and bucket.memory] + if empty: + names = ", ".join(comment_safe(memory.name) for memory in empty) + lines.append(f" # No section lies in {names}. To draw one, add a column with its id as root.") + for region, title in _expansions(buckets, top): # a column of its own, for its width + root, column = _quote(region), _quote(f"{region}_col") + lines.append(f" - {{ id: {column}, root: {root}, width: {COLUMN_WIDTH} }}") + expand.append(f" - {{ region: {root}, to: {column}, title: {_quote(title)} }}") + if expand: + lines += [" expand:", *expand] + return lines + + +def write_yaml( + source: str, + bits: int, + buckets: list[Bucket], + top: Bucket, + regions_from: str = "--region", + notes: tuple[str, ...] = (), +) -> str: + """The config text. `regions_from` says where the memory regions came from, and each of + `notes` becomes a comment line under the header.""" + lines = [ + f"# Generated by `memmap from-elf` from {comment_safe(source)}. Section ranges come from", + ( + f"# the ELF and memory regions from {comment_safe(regions_from)}. Rename, merge or " + "annotate blocks freely." + ), + *(f"# {comment_safe(note)}" for note in notes), + "version: 1", + "", + "figure:", + f" title: {_quote(f'{source} memory map')}", + "", + *_memory_lines(bits, buckets, top), + "", + *_view_lines(buckets, top), + "", + "style:", + " reserved: { show_label: true } # names the Free blocks", + "", + ] + return "\n".join(lines) diff --git a/tests/elf_builder.py b/tests/elf_builder.py index 515e8ca..1b874a9 100644 --- a/tests/elf_builder.py +++ b/tests/elf_builder.py @@ -2,7 +2,7 @@ import struct -from memmap.elf import SHF_ALLOC +from memmap.from_elf.elf import SHF_ALLOC PROGBITS = 1 @@ -54,3 +54,24 @@ def segment_header(offset, virtual, physical, file_size, memory_size): def code(name, address, size, offset=0x1000): """An allocated PROGBITS section header tuple for make_elf.""" return (name, PROGBITS, SHF_ALLOC, address, size, offset) + + +def rebuild_snapshot(snapshot): + """An ELF header snapshot from tests/fixtures/elf_snapshots/ as ELF bytes. make_elf adds its + own section name table.""" + sections = [ + (s["name"], s["type"], s["flags"], int(s["address"], 16), s["size"], s["offset"]) + for s in snapshot["sections"] + if s["name"] != ".shstrtab" + ] + segments = [ + (s["offset"], int(s["virtual"], 16), int(s["physical"], 16), s["file_size"], s["memory_size"]) + for s in snapshot["segments"] + ] + return make_elf( + sections, + segments, + bits=snapshot["bits"], + little=snapshot["little_endian"], + file_type=snapshot["file_type"], + ) diff --git a/tests/fixtures/elf_snapshots/refresh.py b/tests/fixtures/elf_snapshots/refresh.py index ea980d7..1826d26 100644 --- a/tests/fixtures/elf_snapshots/refresh.py +++ b/tests/fixtures/elf_snapshots/refresh.py @@ -2,7 +2,7 @@ A snapshot keeps only what `memmap from-elf` reads: the section headers and the loadable segments, plus the `--region` specs to import with. No code or data from the firmware is kept, -so the snapshots are small and carry none of the firmware's own files. docs/elf-testing.md +so the snapshots are small and carry none of the firmware's own files. docs/from-elf.md explains how the tests use them. uv run python tests/fixtures/elf_snapshots/refresh.py # all snapshots @@ -21,7 +21,7 @@ import urllib.request from pathlib import Path -from memmap.elf import PT_LOAD, read_elf +from memmap.from_elf.elf import PT_LOAD, read_elf HERE = Path(__file__).parent ROOT = HERE.parent.parent.parent diff --git a/tests/fixtures/elf_src/build.sh b/tests/fixtures/elf_src/build.sh index 028a311..6c842c5 100755 --- a/tests/fixtures/elf_src/build.sh +++ b/tests/fixtures/elf_src/build.sh @@ -1,6 +1,7 @@ #!/bin/sh # Rebuilds the real ELF fixtures in tests/fixtures/. Tests use the committed files. # firmware.elf 32-bit little-endian, from arm-none-eabi-gcc +# firmware.map its GNU ld map, linked from an object file so that paths stay relative # firmware_be.elf 32-bit big-endian, the same code built with -mbig-endian # firmware64.elf 64-bit little-endian, firmware.elf converted by GNU objcopy # firmware64_be.elf 64-bit big-endian, firmware_be.elf converted by GNU objcopy @@ -11,7 +12,9 @@ set -eu cd "$(dirname "$0")" OBJCOPY=${OBJCOPY:-objcopy} FLAGS="-mcpu=cortex-m4 -mthumb -Os -nostdlib -ffreestanding -Wl,--build-id=none -T firmware.ld" -arm-none-eabi-gcc $FLAGS -o ../firmware.elf firmware.c +arm-none-eabi-gcc $FLAGS -c -o firmware.o firmware.c +arm-none-eabi-gcc $FLAGS -Wl,-Map=../firmware.map -o ../firmware.elf firmware.o +rm firmware.o arm-none-eabi-gcc $FLAGS -mbig-endian -o ../firmware_be.elf firmware.c arm-none-eabi-strip --strip-debug ../firmware.elf ../firmware_be.elf # objcopy moves the empty ._stack load address and says so; that segment has no bytes in flash. diff --git a/tests/fixtures/firmware.map b/tests/fixtures/firmware.map new file mode 100644 index 0000000..2ac9020 --- /dev/null +++ b/tests/fixtures/firmware.map @@ -0,0 +1,86 @@ + +There are no discarded input sections + +Memory Configuration + +Name Origin Length Attributes +FLASH 0x08000000 0x00010000 xr +RAM 0x20000000 0x00005000 xrw +*default* 0x00000000 0xffffffff + +Linker script and memory map + +LOAD firmware.o + +.isr_vector 0x08000000 0xc0 + *(.isr_vector) + .isr_vector 0x08000000 0xc0 firmware.o + +.text 0x080000c0 0x84 + *(.text*) + .text 0x080000c0 0x84 firmware.o + 0x080000c0 reset_handler + +.glue_7 0x08000144 0x0 + .glue_7 0x08000144 0x0 linker stubs + +.glue_7t 0x08000144 0x0 + .glue_7t 0x08000144 0x0 linker stubs + +.vfp11_veneer 0x08000144 0x0 + .vfp11_veneer 0x08000144 0x0 linker stubs + +.v4_bx 0x08000144 0x0 + .v4_bx 0x08000144 0x0 linker stubs + +.iplt 0x08000144 0x0 + .iplt 0x08000144 0x0 firmware.o + +.rodata 0x08000144 0x4de + 0x08000144 . = ALIGN (0x4) + *(.rodata*) + .rodata 0x08000144 0x4de firmware.o + +.rel.dyn 0x08000624 0x0 + .rel.iplt 0x08000624 0x0 firmware.o + +.ARM.exidx + *(.ARM.exidx*) + +.data 0x20000000 0x40 load address 0x08000622 + 0x20000000 . = ALIGN (0x4) + 0x20000000 _sdata = . + *(.data*) + .data 0x20000000 0x40 firmware.o + 0x20000000 counters + 0x20000040 _edata = . + 0x08000622 _sidata = LOADADDR (.data) + +.igot.plt 0x20000040 0x0 load address 0x08000662 + .igot.plt 0x20000040 0x0 firmware.o + +.bss 0x20000040 0x1000 load address 0x08000662 + 0x20000040 . = ALIGN (0x4) + 0x20000040 _sbss = . + *(.bss*) + .bss 0x20000040 0x1000 firmware.o + 0x20000040 rx_buffer + *(COMMON) + 0x20001040 _ebss = . + +._stack 0x20001040 0x800 load address 0x08000662 + 0x20001040 . = ALIGN (0x8) + 0x20001840 . = (. + 0x800) + *fill* 0x20001040 0x800 + 0x20001840 _estack = . +OUTPUT(firmware.elf elf32-littlearm) +LOAD linker stubs + +.comment 0x00000000 0x46 + .comment 0x00000000 0x46 firmware.o + 0x47 (size before relaxing) + +.ARM.attributes + 0x00000000 0x2e + .ARM.attributes + 0x00000000 0x2e firmware.o diff --git a/tests/fixtures/linker_snapshots/iar6_stm32f103.map b/tests/fixtures/linker_snapshots/iar6_stm32f103.map new file mode 100644 index 0000000..62cbc7f --- /dev/null +++ b/tests/fixtures/linker_snapshots/iar6_stm32f103.map @@ -0,0 +1,12 @@ +Excerpt of https://raw.githubusercontent.com/Seeed-Studio/DSO_Nano/1feaea30263e1383a93cd345724396bf7a7c6aed/DS0201_APP/project/EWARM/FLASH/List/DS0201_APP.map + +# IAR ELF Linker V6.10.2.52244/W32 for ARM 22/Feb/2011 19:04:23 # + +*** PLACEMENT SUMMARY +*** + +"A1": place at 0x0800c000 { ro section .intvec }; +"P1": place in [from 0x0800c000 to 0x0801ffff] { ro }; +"P2": place in [from 0x20001800 to 0x20004fff] { rw, block CSTACK, block HEAP }; + + Section Kind Address Size Object diff --git a/tests/fixtures/linker_snapshots/iar8_rt685.map b/tests/fixtures/linker_snapshots/iar8_rt685.map new file mode 100644 index 0000000..33b4c5a --- /dev/null +++ b/tests/fixtures/linker_snapshots/iar8_rt685.map @@ -0,0 +1,32 @@ +Excerpt of https://raw.githubusercontent.com/JayHeng/NXP-MCUBootUtility/bab2c4d3d7e691c4415cada60934ab80fa073b44/apps/NXP_MIMXRT685-EVK_Rev.E/led_blinky_0x08001000_fdcb_iar.map + +# IAR ELF Linker V8.40.2.214/W32 for ARM 17/Feb/2020 15:25:50 + +*** PLACEMENT SUMMARY +*** + +"A0": place at address 0x800'0400 { ro section .flash_conf }; +"A1": place at address 0x800'1000 { ro section .intvec }; +"P1": place in [from 0x800'1000 to 0x800'112f] | + [from 0x800'1130 to 0x81f'ffff] { ro }; +"P3": place in [from 0x2008'0000 to 0x201f'fbff] { + section CodeQuickAccess, section DataQuickAccess }; +define block RW { rw }; +"P4": place in [from 0x2008'0000 to 0x201f'fbff] { block RW }; +define block ZI { zi }; +"P5": place in [from 0x2008'0000 to 0x201f'fbff] { block ZI }; +define block CSTACK with size = 1K, alignment = 8 { }; +"P7": place in [from 0x201f'fc00 to 0x201f'ffff] { block CSTACK }; +initialize by copy { + rw, section .textrw, section CodeQuickAccess, section DataQuickAccess }; +keep { section .flash_conf }; + +No sections matched the following patterns: + + section DataQuickAccess in "P3-P5" + section m_interrupts_ram in "P2" + section m_usb_bdt in "P8" + section m_usb_global in "P9" + + + Section Kind Address Size Object diff --git a/tests/fixtures/linker_snapshots/iar8_stm32f030.map b/tests/fixtures/linker_snapshots/iar8_stm32f030.map new file mode 100644 index 0000000..80125f0 --- /dev/null +++ b/tests/fixtures/linker_snapshots/iar8_stm32f030.map @@ -0,0 +1,16 @@ +Excerpt of https://raw.githubusercontent.com/SergeyMax/SolderingStation/68c87792b5ed257ba0caaac4c897abc205cd0a69/Firmware/EWARM/Firmware/List/Firmware.map + +# IAR ELF Linker V8.20.1.14183/W32 for ARM 23/Apr/2019 20:37:23 + +*** PLACEMENT SUMMARY +*** + +"A0": place at 0x08000000 { ro section .intvec }; +"P1": place in [from 0x08000000 to 0x08007fff] { ro }; +define block CSTACK with size = 1K, alignment = 8 { }; +define block HEAP with size = 512, alignment = 8 { }; +"P2": place in [from 0x20000000 to 0x20000fff] { + rw, block CSTACK, block HEAP }; +initialize by copy { rw }; + + Section Kind Address Size Object diff --git a/tests/fixtures/linker_snapshots/iar8_stm32f407_ccm.map b/tests/fixtures/linker_snapshots/iar8_stm32f407_ccm.map new file mode 100644 index 0000000..90fd1cf --- /dev/null +++ b/tests/fixtures/linker_snapshots/iar8_stm32f407_ccm.map @@ -0,0 +1,17 @@ +Excerpt of https://raw.githubusercontent.com/Andy-Big/MKS-DLP-alter/a26593ad8b9ac251a6cdc8b4420646ff02667b7b/MKSDLP_Alter_debug_2_0/List/mksDLP.map + +# IAR ELF Linker V8.32.1.169/W32 for ARM 29/May/2021 03:23:38 + +*** PLACEMENT SUMMARY +*** + +"A0": place at 0x801'0000 { ro section .intvec }; +"P1": place in [from 0x801'0000 to 0x807'ffff] { ro }; +define block CSTACK with size = 20K, alignment = 8 { }; +define block HEAP with size = 8K, alignment = 8 { }; +"P2": place in [from 0x2000'0000 to 0x2001'ffff] { + rw, block CSTACK, block HEAP }; +"P3": place in [from 0x1000'0000 to 0x1000'ffff] { rw section CCMRAM }; +initialize by copy { rw }; + + Section Kind Address Size Object diff --git a/tests/fixtures/linker_snapshots/iar9_stm32f407.map b/tests/fixtures/linker_snapshots/iar9_stm32f407.map new file mode 100644 index 0000000..0146ef2 --- /dev/null +++ b/tests/fixtures/linker_snapshots/iar9_stm32f407.map @@ -0,0 +1,24 @@ +Excerpt of https://raw.githubusercontent.com/ApratimPhadke/KRONOS/c15e8d004f4156a60f7dbed7ceb27c92e44d2fb2/Debug/List/kronos.map + +# IAR ELF Linker V9.70.4.588/W64 for ARM 23/May/2026 00:33:15 + +*** PLACEMENT SUMMARY +*** + +"A0": place at address 0x800'0000 { ro section .intvec }; +"P1": place in [from 0x800'0000 to 0x80f'ffff] { ro }; +define block CSTACK with size = 8K, alignment = 8 { }; +define block HEAP with size = 8K, alignment = 8 { }; +"P4": place in [from 0x2000'0000 to 0x2001'bfff] | + [from 0x2001'c000 to 0x2001'ffff] { + rw, block CSTACK, block HEAP }; +initialize by copy { rw }; + +No sections matched the following patterns: + + ro section application_specific_ro in "P2" + section .backup_sram in "P5" + section .ccram in "P3" + + + Section Kind Address Alignment Size Object diff --git a/tests/fixtures/linker_snapshots/iar_stm8.map b/tests/fixtures/linker_snapshots/iar_stm8.map new file mode 100644 index 0000000..93d1857 --- /dev/null +++ b/tests/fixtures/linker_snapshots/iar_stm8.map @@ -0,0 +1,44 @@ +Excerpt of https://raw.githubusercontent.com/Emile666/stc1000_stm8/96cc63ffbaf0012f32bf441f4bf91d71f1e20182/src/Debug/List/stc1000p.map + +# IAR ELF Linker V3.10.1.201 for STM8 17/Feb/2021 21:25:43 + +*** PLACEMENT SUMMARY +*** + +"A0": place at start of [0x000000-0x0000ff] { rw section .vregs }; +"A1": place at end of [0x000000-0x0003ff] { block CSTACK }; +"P2": place in [from 0x000000 to 0x0003ff] { + block HEAP, rw section __DLIB_PERTHREAD, rw section .far.bss, + rw section .far.data, rw section .far.noinit, + rw section .far_func.textrw, rw section .huge.bss, + rw section .huge.data, rw section .huge.noinit, + rw section .huge_func.textrw, rw section .iar.dynexit, + rw section .near.bss, rw section .near.data, rw section .near.noinit, + rw section .near_func.textrw }; +"A2": place at start of [0x008000-0x009fff] { block INTVEC }; +"P3": place in [from 0x008000 to 0x009fff] { + ro section __DLIB_PERTHREAD_init, ro section .far.data_init, + ro section .far_func.textrw_init, ro section .huge.data_init, + ro section .huge_func.textrw_init, ro section .iar.init_table, + ro section .init_array, ro section .near.data_init, + ro section .near.rodata, ro section .near_func.text, + ro section .near_func.textrw_init, ro section .tiny.data_init, + ro section .tiny.rodata_init }; +"P4": place in [from 0x008000 to 0x009fff] { + ro section .far.rodata, ro section .far_func.text }; +"P5": place in [from 0x008000 to 0x009fff] { + ro section .huge.rodata, ro section .huge_func.text }; +"P6": place in [from 0x004000 to 0x00407f] { section .eeprom.noinit }; +"P7": place in [from 0x004000 to 0x00407f] { section .eeprom.data }; +"P8": place in [from 0x004000 to 0x00407f] { section .eeprom.rodata }; +do not initialize { + rw section .eeprom.noinit, rw section .far.noinit, rw section .huge.noinit, + rw section .near.noinit, rw section .tiny.noinit, rw section .vregs }; +initialize by copy { + rw section .far.bss, rw section .far.data, rw section .far_func.textrw, + rw section .huge.bss, rw section .huge.data, rw section .huge_func.textrw, + rw section .iar.dynexit, rw section .near.bss, rw section .near.data, + rw section .near_func.textrw, rw section .tiny.bss, rw section .tiny.data, + ro section .tiny.rodata }; + + Section Kind Address Size Object diff --git a/tests/fixtures/linker_snapshots/index.json b/tests/fixtures/linker_snapshots/index.json new file mode 100644 index 0000000..f62e13d --- /dev/null +++ b/tests/fixtures/linker_snapshots/index.json @@ -0,0 +1,93 @@ +[ + { + "name": "zephyr_nrf52dk.map", + "about": "Zephyr on nRF52832: Windows line ends, IDT_LIST ending one byte below the top", + "url": "https://raw.githubusercontent.com/9enki/nrf52dk_blinky/264f67e741e9d2c86f1137b8bf16c6e9c0509d7e/build/zephyr/zephyr.map", + "sha256": "36884dbf008db23afbda9e85aee1c17d549653eaa5cfa4dd088177eb6594e6c0", + "kept": "gnu" + }, + { + "name": "zephyr_esp32.map", + "about": "Zephyr on ESP32 (Xtensa), seven regions printed at 64-bit width", + "url": "https://raw.githubusercontent.com/faarodriguezpi/esp32_threads_i2c_gpio/1becd99e164caac04780676c3f9ceb380e94fe10/build/zephyr/zephyr.map", + "sha256": "3d0a023c6af6a309b5596211efb251eab7da1be067ae592ddd2e0c35fe3c938b", + "kept": "gnu" + }, + { + "name": "zephyr_qemu_x86.map", + "about": "Zephyr on x86, one RAM region and an IDT_LIST near the top of memory", + "url": "https://raw.githubusercontent.com/granikus/challenge-zephyr-hello/4baeb53ab2fc012cb6eefa9e27d2864c1186a356/build/zephyr/zephyr.map", + "sha256": "cd55a638fcc3c29b9342443aa1fb1afce74b471d3e29908441c427ae7becbcfa", + "kept": "gnu" + }, + { + "name": "keil_f103_ac505.map", + "about": "Keil, Arm Compiler 5.05 on STM32F103: Base: and Max:", + "url": "https://raw.githubusercontent.com/yohanes-erwin/stm32f103-keil/53967360e8acf6d0bd495a03bf252202dc902aa7/dma/Listings/dma.map", + "sha256": "4ec5cd3b7d37ad62f59071272101621762b66ccf2c1a4c0447f0120d03a7d7e8", + "kept": "armlink" + }, + { + "name": "keil_f103_ac506.map", + "about": "Keil, Arm Compiler 5.06 on STM32F103: Exec base: and Load base:", + "url": "https://raw.githubusercontent.com/shengqing1/openmv-stm32/5b658bced8abda61daee102530fe6ce9ede03eed/Listings/Project.map", + "sha256": "946732762ae7d1a1177e735b6ee94bf06545f958accbd17f245e1161437f5aac", + "kept": "armlink" + }, + { + "name": "keil_f407_compressed.map", + "about": "Keil, Arm Compiler 5.06 on STM32F407: compressed RW data, and an empty CCM region", + "url": "https://raw.githubusercontent.com/Caesar037/STM32_Intelligent_Lock/b7f4329a3402a85b2fd9bdaa3be9ec2345527b9e/Listings/Template.map", + "sha256": "4f883a94949bbd74c1ea8e5286030d5114faf35cb6ca664839d708bde50be8d9", + "kept": "armlink" + }, + { + "name": "keil_no_scatter.map", + "about": "Keil, Arm Compiler 5.06 without a scatter file: regions with no Max", + "url": "https://raw.githubusercontent.com/PacktPublishing/ARM-Assembly-Language-from-Ground-Up-1/567c112da1f3374594da0fff4c6d50d2ff5ad2d7/Swap/Listings/Swap.map", + "sha256": "9ec91b56d695fd7a5611daf8a106a93b96d9212fed7b74a269eca882cd45c46d", + "kept": "armlink" + }, + { + "name": "iar6_stm32f103.map", + "about": "IAR 6.10: plain hex, an application above a bootloader", + "url": "https://raw.githubusercontent.com/Seeed-Studio/DSO_Nano/1feaea30263e1383a93cd345724396bf7a7c6aed/DS0201_APP/project/EWARM/FLASH/List/DS0201_APP.map", + "sha256": "74d9f3a40a886eb00b790fb8b44dee945ba334c956cafa9490339cfb9d20a787", + "kept": "iar" + }, + { + "name": "iar8_stm32f030.map", + "about": "IAR 8.20: a ROM and a RAM placement", + "url": "https://raw.githubusercontent.com/SergeyMax/SolderingStation/68c87792b5ed257ba0caaac4c897abc205cd0a69/Firmware/EWARM/Firmware/List/Firmware.map", + "sha256": "9a4a27f28fcb23473feeedfee89ebeec480039e0cf0cea48241749dad53208ab", + "kept": "iar" + }, + { + "name": "iar8_stm32f407_ccm.map", + "about": "IAR 8.32: digit groups with ', and RAM in CCM and SRAM", + "url": "https://raw.githubusercontent.com/Andy-Big/MKS-DLP-alter/a26593ad8b9ac251a6cdc8b4420646ff02667b7b/MKSDLP_Alter_debug_2_0/List/mksDLP.map", + "sha256": "beefaa2a33af1b9275dad99eb2546dd5b323cbca94a878ebf909302c15fc9af5", + "kept": "iar" + }, + { + "name": "iar8_rt685.map", + "about": "IAR 8.40 on i.MX RT685: unions of ranges, and a stack placed apart", + "url": "https://raw.githubusercontent.com/JayHeng/NXP-MCUBootUtility/bab2c4d3d7e691c4415cada60934ab80fa073b44/apps/NXP_MIMXRT685-EVK_Rev.E/led_blinky_0x08001000_fdcb_iar.map", + "sha256": "17d1e8393cac95314ea7c02ea276389d410b884928ad896d6825e2a79daa4d12", + "kept": "iar" + }, + { + "name": "iar9_stm32f407.map", + "about": "IAR 9.70: RAM as two adjoining ranges, and placements that matched nothing", + "url": "https://raw.githubusercontent.com/ApratimPhadke/KRONOS/c15e8d004f4156a60f7dbed7ceb27c92e44d2fb2/Debug/List/kronos.map", + "sha256": "c08313febf36ff65276afe93790a3919aabe49b801aedf0a9e9277019fc9350a", + "kept": "iar" + }, + { + "name": "iar_stm8.map", + "about": "IAR for STM8: [0x0-0xff] ranges, and EEPROM holding named sections only", + "url": "https://raw.githubusercontent.com/Emile666/stc1000_stm8/96cc63ffbaf0012f32bf441f4bf91d71f1e20182/src/Debug/List/stc1000p.map", + "sha256": "a8f33a6e12b5f1f2e9f07f73859bd68f7a4d32c7847f319ed84a1b13fb13a31f", + "kept": "iar" + } +] diff --git a/tests/fixtures/linker_snapshots/keil_f103_ac505.map b/tests/fixtures/linker_snapshots/keil_f103_ac505.map new file mode 100644 index 0000000..6db69d4 --- /dev/null +++ b/tests/fixtures/linker_snapshots/keil_f103_ac505.map @@ -0,0 +1,6 @@ +Excerpt of https://raw.githubusercontent.com/yohanes-erwin/stm32f103-keil/53967360e8acf6d0bd495a03bf252202dc902aa7/dma/Listings/dma.map + +Memory Map of the image + Load Region LR_IROM1 (Base: 0x08000000, Size: 0x0000158c, Max: 0x00010000, ABSOLUTE) + Execution Region ER_IROM1 (Base: 0x08000000, Size: 0x00001538, Max: 0x00010000, ABSOLUTE) + Execution Region RW_IRAM1 (Base: 0x20000000, Size: 0x000006d8, Max: 0x00005000, ABSOLUTE) diff --git a/tests/fixtures/linker_snapshots/keil_f103_ac506.map b/tests/fixtures/linker_snapshots/keil_f103_ac506.map new file mode 100644 index 0000000..2648627 --- /dev/null +++ b/tests/fixtures/linker_snapshots/keil_f103_ac506.map @@ -0,0 +1,6 @@ +Excerpt of https://raw.githubusercontent.com/shengqing1/openmv-stm32/5b658bced8abda61daee102530fe6ce9ede03eed/Listings/Project.map + +Memory Map of the image + Load Region LR_IROM1 (Base: 0x08000000, Size: 0x00001b1c, Max: 0x00010000, ABSOLUTE) + Execution Region ER_IROM1 (Exec base: 0x08000000, Load base: 0x08000000, Size: 0x00001ad0, Max: 0x00010000, ABSOLUTE) + Execution Region RW_IRAM1 (Exec base: 0x20000000, Load base: 0x08001ad0, Size: 0x000007d0, Max: 0x00005000, ABSOLUTE) diff --git a/tests/fixtures/linker_snapshots/keil_f407_compressed.map b/tests/fixtures/linker_snapshots/keil_f407_compressed.map new file mode 100644 index 0000000..d38481e --- /dev/null +++ b/tests/fixtures/linker_snapshots/keil_f407_compressed.map @@ -0,0 +1,7 @@ +Excerpt of https://raw.githubusercontent.com/Caesar037/STM32_Intelligent_Lock/b7f4329a3402a85b2fd9bdaa3be9ec2345527b9e/Listings/Template.map + +Memory Map of the image + Load Region LR_IROM1 (Base: 0x08000000, Size: 0x00008058, Max: 0x00080000, ABSOLUTE, COMPRESSED[0x00006214]) + Execution Region ER_IROM1 (Exec base: 0x08000000, Load base: 0x08000000, Size: 0x00005434, Max: 0x00080000, ABSOLUTE) + Execution Region RW_IRAM2 (Exec base: 0x10000000, Load base: 0x08006214, Size: 0x00000000, Max: 0x00010000, ABSOLUTE) + Execution Region RW_IRAM1 (Exec base: 0x20000000, Load base: 0x08005434, Size: 0x00003190, Max: 0x00020000, ABSOLUTE, COMPRESSED[0x00000de0]) diff --git a/tests/fixtures/linker_snapshots/keil_no_scatter.map b/tests/fixtures/linker_snapshots/keil_no_scatter.map new file mode 100644 index 0000000..907762b --- /dev/null +++ b/tests/fixtures/linker_snapshots/keil_no_scatter.map @@ -0,0 +1,7 @@ +Excerpt of https://raw.githubusercontent.com/PacktPublishing/ARM-Assembly-Language-from-Ground-Up-1/567c112da1f3374594da0fff4c6d50d2ff5ad2d7/Swap/Listings/Swap.map + +Memory Map of the image + Load Region LR_1 (Base: 0x08000000, Size: 0x0000023c, Max: 0xffffffff, ABSOLUTE) + Execution Region ER_RO (Exec base: 0x08000000, Load base: 0x08000000, Size: 0x0000023c, Max: 0xffffffff, ABSOLUTE) + Execution Region ER_RW (Exec base: 0x20000000, Load base: 0x0800023c, Size: 0x00000000, Max: 0xffffffff, ABSOLUTE) + Execution Region ER_ZI (Exec base: 0x20000000, Load base: 0x0800023c, Size: 0x00000400, Max: 0xffffffff, ABSOLUTE) diff --git a/tests/fixtures/linker_snapshots/refresh.py b/tests/fixtures/linker_snapshots/refresh.py new file mode 100644 index 0000000..a5e0618 --- /dev/null +++ b/tests/fixtures/linker_snapshots/refresh.py @@ -0,0 +1,185 @@ +"""Rewrite the linker map snapshots in this folder from the real files in build/linker/. + +A snapshot keeps only the lines that `memmap from-elf --map` reads: the Memory Configuration +table of a GNU ld map, the region lines of an armlink map, the placement summary of an IAR map. +Each starts with where it came from. index.json lists them with the SHA-256 of the full file. +docs/from-elf.md explains how the tests use them. + + uv run python tests/fixtures/linker_snapshots/refresh.py # all snapshots + uv run python tests/fixtures/linker_snapshots/refresh.py --download # fetch missing files + +After a refresh, regenerate the expected regions with +MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_linker.py and review their diff. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import urllib.request +from pathlib import Path + +HERE = Path(__file__).parent +ROOT = HERE.parent.parent.parent +SOURCE_DIR = ROOT / "build" / "linker" +GITHUB = "https://raw.githubusercontent.com/" + +# name: where the map comes from, what it shows, and which reader's lines to keep of it. +MANIFEST: dict[str, dict[str, object]] = { + "zephyr_nrf52dk.map": { + "url": GITHUB + "9enki/nrf52dk_blinky/264f67e741e9d2c86f1137b8bf16c6e9c0509d7e" + "/build/zephyr/zephyr.map", + "about": "Zephyr on nRF52832: Windows line ends, IDT_LIST ending one byte below the top", + "keep": "gnu", + }, + "zephyr_esp32.map": { + "url": GITHUB + "faarodriguezpi/esp32_threads_i2c_gpio/" + "1becd99e164caac04780676c3f9ceb380e94fe10/build/zephyr/zephyr.map", + "about": "Zephyr on ESP32 (Xtensa), seven regions printed at 64-bit width", + "keep": "gnu", + }, + "zephyr_qemu_x86.map": { + "url": GITHUB + "granikus/challenge-zephyr-hello/4baeb53ab2fc012cb6eefa9e27d2864c1186a356" + "/build/zephyr/zephyr.map", + "about": "Zephyr on x86, one RAM region and an IDT_LIST near the top of memory", + "keep": "gnu", + }, + "keil_f103_ac505.map": { + "url": GITHUB + "yohanes-erwin/stm32f103-keil/53967360e8acf6d0bd495a03bf252202dc902aa7" + "/dma/Listings/dma.map", + "about": "Keil, Arm Compiler 5.05 on STM32F103: Base: and Max:", + "keep": "armlink", + }, + "keil_f103_ac506.map": { + "url": GITHUB + "shengqing1/openmv-stm32/5b658bced8abda61daee102530fe6ce9ede03eed" + "/Listings/Project.map", + "about": "Keil, Arm Compiler 5.06 on STM32F103: Exec base: and Load base:", + "keep": "armlink", + }, + "keil_f407_compressed.map": { + "url": GITHUB + "Caesar037/STM32_Intelligent_Lock/b7f4329a3402a85b2fd9bdaa3be9ec2345527b9e" + "/Listings/Template.map", + "about": "Keil, Arm Compiler 5.06 on STM32F407: compressed RW data, and an empty CCM region", + "keep": "armlink", + }, + "keil_no_scatter.map": { + "url": GITHUB + "PacktPublishing/ARM-Assembly-Language-from-Ground-Up-1/" + "567c112da1f3374594da0fff4c6d50d2ff5ad2d7/Swap/Listings/Swap.map", + "about": "Keil, Arm Compiler 5.06 without a scatter file: regions with no Max", + "keep": "armlink", + }, + "iar6_stm32f103.map": { + "url": GITHUB + "Seeed-Studio/DSO_Nano/1feaea30263e1383a93cd345724396bf7a7c6aed" + "/DS0201_APP/project/EWARM/FLASH/List/DS0201_APP.map", + "about": "IAR 6.10: plain hex, an application above a bootloader", + "keep": "iar", + }, + "iar8_stm32f030.map": { + "url": GITHUB + "SergeyMax/SolderingStation/68c87792b5ed257ba0caaac4c897abc205cd0a69" + "/Firmware/EWARM/Firmware/List/Firmware.map", + "about": "IAR 8.20: a ROM and a RAM placement", + "keep": "iar", + }, + "iar8_stm32f407_ccm.map": { + "url": GITHUB + "Andy-Big/MKS-DLP-alter/a26593ad8b9ac251a6cdc8b4420646ff02667b7b" + "/MKSDLP_Alter_debug_2_0/List/mksDLP.map", + "about": "IAR 8.32: digit groups with ', and RAM in CCM and SRAM", + "keep": "iar", + }, + "iar8_rt685.map": { + "url": GITHUB + "JayHeng/NXP-MCUBootUtility/bab2c4d3d7e691c4415cada60934ab80fa073b44" + "/apps/NXP_MIMXRT685-EVK_Rev.E/led_blinky_0x08001000_fdcb_iar.map", + "about": "IAR 8.40 on i.MX RT685: unions of ranges, and a stack placed apart", + "keep": "iar", + }, + "iar9_stm32f407.map": { + "url": GITHUB + "ApratimPhadke/KRONOS/c15e8d004f4156a60f7dbed7ceb27c92e44d2fb2" + "/Debug/List/kronos.map", + "about": "IAR 9.70: RAM as two adjoining ranges, and placements that matched nothing", + "keep": "iar", + }, + "iar_stm8.map": { + "url": GITHUB + "Emile666/stc1000_stm8/96cc63ffbaf0012f32bf441f4bf91d71f1e20182" + "/src/Debug/List/stc1000p.map", + "about": "IAR for STM8: [0x0-0xff] ranges, and EEPROM holding named sections only", + "keep": "iar", + }, +} + + +def _gnu(lines: list[str]) -> list[str]: + """The Memory Configuration table of a GNU ld map.""" + start = next(i for i, line in enumerate(lines) if line.strip() == "Memory Configuration") + end = next(i for i, line in enumerate(lines) if line.startswith("Linker script and memory map")) + return lines[start:end] + + +def _armlink(lines: list[str]) -> list[str]: + """The heading and region lines of an armlink map's memory map.""" + keep = ("Memory Map of the image", "Load Region ", "Execution Region ") + return [line for line in lines if any(word in line for word in keep)] + + +def _iar(lines: list[str]) -> list[str]: + """The linker version line and the placement summary of an IAR map.""" + version = next(line for line in lines if "IAR ELF Linker" in line) + start = next(i for i, line in enumerate(lines) if line.startswith("*** PLACEMENT SUMMARY")) + end = next(i for i in range(start, len(lines)) if re.match(r"\s+Section\s+Kind", lines[i])) + return [version, "", *lines[start : end + 1]] + + +EXCERPTS = {"gnu": _gnu, "armlink": _armlink, "iar": _iar} + + +def snapshot(entry: dict[str, object], data: bytes) -> str: + """The snapshot text of one map: where it came from, as a plain line the readers skip, + then the lines its reader uses.""" + lines = data.decode("utf-8", "replace").replace("\r\n", "\n").split("\n") + kept = EXCERPTS[str(entry["keep"])](lines) + return "\n".join([f"Excerpt of {entry['url']}", "", *kept]).rstrip("\n") + "\n" + + +def main() -> int: + """Refresh every snapshot whose file is in build/linker/. Returns the exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--download", action="store_true", help="fetch missing files first") + args = parser.parse_args() + index = [] + missing = [] + for name, entry in MANIFEST.items(): + source = SOURCE_DIR / name + if not source.exists() and args.download: + print(f"downloading {name}") + source.parent.mkdir(parents=True, exist_ok=True) + urllib.request.urlretrieve(str(entry["url"]), source) + if not source.exists(): + missing.append(name) + continue + data = source.read_bytes() + target = HERE / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(snapshot(entry, data), encoding="utf-8", newline="\n") + index.append( + { + "name": name, + "about": entry["about"], + "url": entry["url"], + "sha256": hashlib.sha256(data).hexdigest(), + "kept": entry["keep"], + } + ) + print(f"wrote {name}") + for name in missing: + print(f"skipped {name}: build/linker/{name} is missing", file=sys.stderr) + if missing: + return 1 + text = json.dumps(index, indent=2) + "\n" + (HERE / "index.json").write_text(text, encoding="utf-8", newline="\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/linker_snapshots/zephyr_esp32.map b/tests/fixtures/linker_snapshots/zephyr_esp32.map new file mode 100644 index 0000000..3d9d55b --- /dev/null +++ b/tests/fixtures/linker_snapshots/zephyr_esp32.map @@ -0,0 +1,13 @@ +Excerpt of https://raw.githubusercontent.com/faarodriguezpi/esp32_threads_i2c_gpio/1becd99e164caac04780676c3f9ceb380e94fe10/build/zephyr/zephyr.map + +Memory Configuration + +Name Origin Length Attributes +iram0_0_seg 0x0000000040080000 0x0000000000020000 xr +iram0_2_seg 0x00000000400d0018 0x0000000000330000 xr +dram0_0_seg 0x000000003ffb0000 0x0000000000050000 rw +drom0_0_seg 0x000000003f400010 0x0000000000800000 r +rtc_iram_seg 0x00000000400c0000 0x0000000000002000 xrw +rtc_slow_seg 0x0000000050000000 0x0000000000001000 rw +IDT_LIST 0x000000003ebfe010 0x0000000000002000 rw +*default* 0x0000000000000000 0xffffffffffffffff diff --git a/tests/fixtures/linker_snapshots/zephyr_nrf52dk.map b/tests/fixtures/linker_snapshots/zephyr_nrf52dk.map new file mode 100644 index 0000000..35c0484 --- /dev/null +++ b/tests/fixtures/linker_snapshots/zephyr_nrf52dk.map @@ -0,0 +1,9 @@ +Excerpt of https://raw.githubusercontent.com/9enki/nrf52dk_blinky/264f67e741e9d2c86f1137b8bf16c6e9c0509d7e/build/zephyr/zephyr.map + +Memory Configuration + +Name Origin Length Attributes +FLASH 0x00000000 0x00030000 xr +SRAM 0x20000000 0x00006000 xw +IDT_LIST 0xfffff7ff 0x00000800 xw +*default* 0x00000000 0xffffffff diff --git a/tests/fixtures/linker_snapshots/zephyr_qemu_x86.map b/tests/fixtures/linker_snapshots/zephyr_qemu_x86.map new file mode 100644 index 0000000..57f806f --- /dev/null +++ b/tests/fixtures/linker_snapshots/zephyr_qemu_x86.map @@ -0,0 +1,8 @@ +Excerpt of https://raw.githubusercontent.com/granikus/challenge-zephyr-hello/4baeb53ab2fc012cb6eefa9e27d2864c1186a356/build/zephyr/zephyr.map + +Memory Configuration + +Name Origin Length Attributes +RAM 0x0000000000100000 0x0000000001f00000 xw +IDT_LIST 0x00000000ffff1000 0x0000000000000800 +*default* 0x0000000000000000 0xffffffffffffffff diff --git a/tests/golden/linker/regions.json b/tests/golden/linker/regions.json new file mode 100644 index 0000000..2073fd5 --- /dev/null +++ b/tests/golden/linker/regions.json @@ -0,0 +1,247 @@ +{ + "zephyr_nrf52dk.map": { + "kind": "GNU ld map", + "regions": [ + [ + "FLASH", + "0x0", + "0x30000" + ], + [ + "SRAM", + "0x20000000", + "0x6000" + ], + [ + "IDT_LIST", + "0xFFFFF7FF", + "0x800" + ] + ] + }, + "zephyr_esp32.map": { + "kind": "GNU ld map", + "regions": [ + [ + "iram0_0_seg", + "0x40080000", + "0x20000" + ], + [ + "iram0_2_seg", + "0x400D0018", + "0x330000" + ], + [ + "dram0_0_seg", + "0x3FFB0000", + "0x50000" + ], + [ + "drom0_0_seg", + "0x3F400010", + "0x800000" + ], + [ + "rtc_iram_seg", + "0x400C0000", + "0x2000" + ], + [ + "rtc_slow_seg", + "0x50000000", + "0x1000" + ], + [ + "IDT_LIST", + "0x3EBFE010", + "0x2000" + ] + ] + }, + "zephyr_qemu_x86.map": { + "kind": "GNU ld map", + "regions": [ + [ + "RAM", + "0x100000", + "0x1F00000" + ], + [ + "IDT_LIST", + "0xFFFF1000", + "0x800" + ] + ] + }, + "keil_f103_ac505.map": { + "kind": "armlink map", + "regions": [ + [ + "ER_IROM1", + "0x8000000", + "0x10000" + ], + [ + "RW_IRAM1", + "0x20000000", + "0x5000" + ] + ] + }, + "keil_f103_ac506.map": { + "kind": "armlink map", + "regions": [ + [ + "ER_IROM1", + "0x8000000", + "0x10000" + ], + [ + "RW_IRAM1", + "0x20000000", + "0x5000" + ] + ] + }, + "keil_f407_compressed.map": { + "kind": "armlink map", + "regions": [ + [ + "ER_IROM1", + "0x8000000", + "0x80000" + ], + [ + "RW_IRAM2", + "0x10000000", + "0x10000" + ], + [ + "RW_IRAM1", + "0x20000000", + "0x20000" + ] + ] + }, + "keil_no_scatter.map": { + "kind": "armlink map", + "regions": [ + [ + "ER_RO", + "0x8000000", + null + ], + [ + "ER_RW", + "0x20000000", + null + ], + [ + "ER_ZI", + "0x20000000", + null + ] + ] + }, + "iar6_stm32f103.map": { + "kind": "IAR map", + "regions": [ + [ + "ROM", + "0x800C000", + "0x14000" + ], + [ + "RAM", + "0x20001800", + "0x3800" + ] + ] + }, + "iar8_stm32f030.map": { + "kind": "IAR map", + "regions": [ + [ + "ROM", + "0x8000000", + "0x8000" + ], + [ + "RAM", + "0x20000000", + "0x1000" + ] + ] + }, + "iar8_stm32f407_ccm.map": { + "kind": "IAR map", + "regions": [ + [ + "ROM", + "0x8010000", + "0x70000" + ], + [ + "RAM", + "0x10000000", + "0x10000" + ], + [ + "RAM 2", + "0x20000000", + "0x20000" + ] + ] + }, + "iar8_rt685.map": { + "kind": "IAR map", + "regions": [ + [ + "ROM", + "0x8001000", + "0x1FF000" + ], + [ + "RAM", + "0x20080000", + "0x180000" + ] + ] + }, + "iar9_stm32f407.map": { + "kind": "IAR map", + "regions": [ + [ + "ROM", + "0x8000000", + "0x100000" + ], + [ + "RAM", + "0x20000000", + "0x20000" + ] + ] + }, + "iar_stm8.map": { + "kind": "IAR map", + "regions": [ + [ + "RAM", + "0x0", + "0x400" + ], + [ + "Memory", + "0x4000", + "0x80" + ], + [ + "ROM", + "0x8000", + "0x2000" + ] + ] + } +} diff --git a/tests/test_elf_snapshots.py b/tests/test_elf_snapshots.py index 3c98ba3..9c85ba9 100644 --- a/tests/test_elf_snapshots.py +++ b/tests/test_elf_snapshots.py @@ -1,5 +1,5 @@ """Real firmware layouts through `from-elf`, from the header snapshots in -tests/fixtures/elf_snapshots/. See docs/elf-testing.md. +tests/fixtures/elf_snapshots/. See docs/from-elf.md. Each snapshot is rebuilt into a minimal ELF, imported with its regions, and the config and its drawing are compared with tests/golden/elf/. Regenerate after an intended change with: @@ -12,10 +12,9 @@ import pytest from conftest import ROOT, codes, drawn -from elf_builder import make_elf +from elf_builder import rebuild_snapshot as rebuild -from memmap.elf import read_elf -from memmap.from_elf import import_elf, parse_region_spec +from memmap.from_elf import import_elf, parse_region_spec, read_elf from memmap.pipeline import run from memmap.render.svg import render_svg @@ -30,26 +29,6 @@ def load(name): return json.loads((SNAPSHOTS / f"{name}.json").read_text(encoding="utf-8")) -def rebuild(snapshot): - """The snapshot as ELF bytes. make_elf adds its own section name table.""" - sections = [ - (s["name"], s["type"], s["flags"], int(s["address"], 16), s["size"], s["offset"]) - for s in snapshot["sections"] - if s["name"] != ".shstrtab" - ] - segments = [ - (s["offset"], int(s["virtual"], 16), int(s["physical"], 16), s["file_size"], s["memory_size"]) - for s in snapshot["segments"] - ] - return make_elf( - sections, - segments, - bits=snapshot["bits"], - little=snapshot["little_endian"], - file_type=snapshot["file_type"], - ) - - def compare(golden, text): """Check `text` against a golden file, or rewrite it with MEMMAP_UPDATE_GOLDEN=1.""" if UPDATE: diff --git a/tests/test_from_elf.py b/tests/test_from_elf.py index 85a395b..850980e 100644 --- a/tests/test_from_elf.py +++ b/tests/test_from_elf.py @@ -8,8 +8,8 @@ from elf_builder import PROGBITS, code, make_elf from memmap.cli import app -from memmap.elf import SHF_ALLOC, SHF_TLS, SHT_NOBITS, ElfError, read_elf from memmap.from_elf import ImportProblem, MemorySpec, import_elf, parse_region_spec +from memmap.from_elf.elf import SHF_ALLOC, SHF_TLS, SHT_NOBITS, ElfError, read_elf from memmap.pipeline import run FIXTURES = ROOT / "tests" / "fixtures" diff --git a/tests/test_linker.py b/tests/test_linker.py new file mode 100644 index 0000000..8fd6dc0 --- /dev/null +++ b/tests/test_linker.py @@ -0,0 +1,736 @@ +"""Memory regions from linker maps: `memmap from-elf --map`. + +The readers run on real maps from tests/fixtures/linker_snapshots/ (see docs/from-elf.md), +compared with tests/golden/linker/regions.json. Regenerate after an intended change with: + + MEMMAP_UPDATE_GOLDEN=1 uv run pytest tests/test_linker.py +""" + +import json +import os +import re + +import pytest +from click.testing import CliRunner +from conftest import ROOT +from elf_builder import code, make_elf + +from memmap.cli import app +from memmap.from_elf import ( + ImportProblem, + LinkerFile, + LinkerRegion, + parse_region_spec, + read_map, + resolve_memories, +) + +FIXTURES = ROOT / "tests" / "fixtures" +SNAPSHOTS = FIXTURES / "linker_snapshots" +INDEX = json.loads((SNAPSHOTS / "index.json").read_text(encoding="utf-8")) +ENTRIES = [entry["name"] for entry in INDEX] +GOLDEN = ROOT / "tests" / "golden" / "linker" / "regions.json" +UPDATE = os.environ.get("MEMMAP_UPDATE_GOLDEN") == "1" +SOURCES = ROOT / "build" / "linker" +FIRMWARE = FIXTURES / "firmware.elf" +FIRMWARE_MAP = FIXTURES / "firmware.map" +LLD_HEADER = " VMA LMA Size Align Out In Symbol\n" +runner = CliRunner() + + +def invoke(*args): + return runner.invoke(app, [str(arg) for arg in args]) + + +def summary(linker): + """A map's regions as JSON-friendly rows.""" + return { + "kind": linker.kind, + "regions": [ + [r.name, f"0x{r.start:X}", None if r.size is None else f"0x{r.size:X}"] + for r in linker.regions + ], + } + + +def write(tmp_path, name, text): + file = tmp_path / name + file.write_text(text, encoding="utf-8") + return file + + +def gnu_map(*rows): + """GNU ld map text whose Memory Configuration table has `rows`, each `NAME ORIGIN LENGTH`.""" + lines = ["Memory Configuration", "", "Name Origin Length Attributes"] + lines += [*rows, "*default* 0x00000000 0xffffffff", "", "Linker script and memory map", ""] + return "\n".join(lines) + + +def elf_file(tmp_path, sections, name="fw.elf"): + file = tmp_path / name + file.write_bytes(make_elf(sections)) + return file + + +def cortex_m(tmp_path): + """A small image with code in flash at 0x08000000 and data in RAM at 0x20000000.""" + return elf_file(tmp_path, [code(".text", 0x08000000, 0x1000), code(".data", 0x20000000, 0x100)]) + + +def usage_problem(result): + """The message and hint of the one E090 of a --json run that exited 2.""" + assert result.exit_code == 2, result.output + (diagnostic,) = json.loads(result.stdout)["diagnostics"] + assert diagnostic["code"] == "E090" + return diagnostic["message"], diagnostic["hint"] + + +def warnings(result): + """(code, subject, hint) of each diagnostic of a --json run.""" + return [(d["code"], d["subject"], d["hint"]) for d in json.loads(result.stdout)["diagnostics"]] + + +def header(file): + """The comment lines at the top of a generated config, one string.""" + lines = file.read_text().splitlines() + return "\n".join(line for line in lines[: lines.index("version: 1")]) + + +# Real files + + +def test_snapshots_are_listed(): + assert len(ENTRIES) >= 13 + assert set(ENTRIES) == {path.name for path in SNAPSHOTS.glob("*.map")} + + +def test_snapshots_read_to_the_golden_regions(): + found = {name: summary(read_map(SNAPSHOTS / name)) for name in ENTRIES} + if UPDATE: + GOLDEN.parent.mkdir(parents=True, exist_ok=True) + GOLDEN.write_text(json.dumps(found, indent=2) + "\n", encoding="utf-8", newline="\n") + assert found == json.loads(GOLDEN.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("name", ENTRIES) +def test_full_file_reads_like_its_snapshot(name): + """The excerpt keeps everything the reader needs. Runs only where refresh.py has the real + file in build/linker/.""" + source = SOURCES / name + if not source.exists(): + pytest.skip(f"build/linker/{name} is missing") + assert summary(read_map(source)) == summary(read_map(SNAPSHOTS / name)) + + +# Readers + + +GNU_MAP = """Memory Configuration\r +\r +Name Origin Length Attributes\r +FLASH 0x08000000 0x00010000 xr\r +RAM 0x20000000 0x00005000 xrw !x\r +a_very_long_region_name 0x30000000 0x00000100\r +*default* 0x00000000 0xffffffff\r +\r +Linker script and memory map\r +\r +FAKE 0x40000000 0x00000100\r +""" + + +def test_gnu_map(tmp_path): + linker = read_map(write(tmp_path, "fw.map", GNU_MAP)) + assert linker.kind == "GNU ld map" + assert [(r.name, r.start, r.size) for r in linker.regions] == [ + ("FLASH", 0x08000000, 0x10000), + ("RAM", 0x20000000, 0x5000), + ("a_very_long_region_name", 0x30000000, 0x100), + ] + + +def test_gnu_map_of_a_script_without_memory_has_no_regions(tmp_path): + linker = read_map(write(tmp_path, "fw.map", gnu_map())) + assert (linker.kind, linker.regions) == ("GNU ld map", []) + + +@pytest.mark.parametrize( + "first_line", + [LLD_HEADER, "Address Size Align Out In Symbol\n"], # the second: lld before 8 +) +def test_lld_map_has_no_regions(tmp_path, first_line): + linker = read_map(write(tmp_path, "fw.map", first_line + " 0 0 100 1 .text\n")) + assert (linker.kind, linker.regions) == ("lld map", []) + + +def test_armlink_map_without_a_limit(tmp_path): + text = ( + "Memory Map of the image\n" + " Load Region LR_1 (Base: 0x08000000, Size: 0x0000023c, Max: 0xffffffff, ABSOLUTE)\n" + " Execution Region ER_RO (Exec base: 0x08000000, Load base: 0x08000000, Size: " + "0x0000023c, Max: 0xffffffff, ABSOLUTE)\n" + " Execution Region ARM_LIB_STACK (Base: 0x2001f000, Size: 0x00001000, Max: " + "0x00001000, ABSOLUTE, EMPTY)\n" + ) + linker = read_map(write(tmp_path, "fw.map", text)) + assert [(r.name, r.start, r.size) for r in linker.regions] == [ + ("ER_RO", 0x08000000, None), + ("ARM_LIB_STACK", 0x2001F000, 0x1000), + ] + + +def test_iar_map_kinds_and_merging(tmp_path): + text = """# IAR ELF Linker V9.30.1.335/W64 for ARM +*** PLACEMENT SUMMARY +"A0": place at address 0x800'0000 { ro section .intvec }; +"P1": place in [from 0x800'0000 to 0x80f'ffff] { ro }; +"P2": place in [from 0x2000'0000 to 0x2001'bfff] | + [from 0x2001'c000 to 0x2001'ffff] { rw, block CSTACK }; +"P3": place in [from 0x1000'0000 size 0x1'0000] { block HEAP }; +"P4": place in [0x4000-0x407f] { section .eeprom }; +"P5": place in [from 0x2002'0000 to 0x2002'0fff] { section .noinit }; + Section Kind Address Size Object +"P9": place in [from 0x9000'0000 to 0x9000'ffff] { ro }; +""" + linker = read_map(write(tmp_path, "fw.map", text)) + assert [(r.name, r.start, r.size) for r in linker.regions] == [ + ("Memory", 0x4000, 0x80), + ("ROM", 0x08000000, 0x100000), + ("RAM", 0x10000000, 0x10000), + ("RAM 2", 0x20000000, 0x21000), # the unknown .noinit range adjoins RAM and joins it + ] + + +def test_iar_map_with_fixed_addresses_only_has_no_regions(tmp_path): + text = "# IAR ELF Linker V9.30\n*** PLACEMENT SUMMARY\n\"A0\": place at address 0x800'0000 { ro };\n" + linker = read_map(write(tmp_path, "fw.map", text)) + assert (linker.kind, linker.regions) == ("IAR map", []) + + +def test_iar_map_adjoining_ranges_of_other_kinds_stay_apart(tmp_path): + text = """# IAR ELF Linker V8.40 +*** PLACEMENT SUMMARY +"P1": place in [from 0x0 to 0xfff] { ro }; +"P2": place in [from 0x1000 to 0x1fff] { rw }; +"P3": place in [from 0x2000 to 0x2fff] { rw, block CSTACK }; +""" + linker = read_map(write(tmp_path, "fw.map", text)) + assert [(r.name, r.start, r.size) for r in linker.regions] == [("ROM", 0, 0x1000), ("RAM", 0x1000, 0x2000)] + + +def test_iar_map_reversed_range_is_skipped(tmp_path): + text = "# IAR ELF Linker V8.40\n*** PLACEMENT SUMMARY\n\"P1\": place in [from 0x2000 to 0x1000] { rw };\n" + assert read_map(write(tmp_path, "fw.map", text)).regions == [] + + +def test_iar_map_with_a_bad_number(tmp_path): + text = "# IAR ELF Linker V8.40\n*** PLACEMENT SUMMARY\n\"P1\": place in [from 0x' to 0x'] { rw };\n" + with pytest.raises(ImportProblem, match="cannot read the address \"0x'\" in the placement summary"): + read_map(write(tmp_path, "fw.map", text)) + + +@pytest.mark.parametrize( + ("text", "kind"), + [ + (LLD_HEADER, "lld map"), + (gnu_map("FLASH 0x0 0x1000 xr"), "GNU ld map"), # the table on the first line + ], +) +def test_maps_with_a_byte_order_mark(tmp_path, text, kind): + file = tmp_path / "fw.map" + file.write_bytes(b"\xef\xbb\xbf" + text.encode()) + assert read_map(file).kind == kind + + +NOT_MAPS = { + "memory.x": ( + "MEMORY\n{\n FLASH : ORIGIN = 0, LENGTH = 1M\n}\n", + "memory.x is a linker script, which memmap does not read", + ("pass the map file the linker writes, or give each region with --region ID=START:SIZE " + "from the script's MEMORY block"), + ), + "fw.ld": ( + "/* boot */\nSECTIONS {\n .text : { *(.text) }\n}\n", + "fw.ld is a linker script, which memmap does not read", + "pass the map file the linker writes, or give each region with --region ID=START:SIZE", + ), + "notes.txt": ("/* MEMORY { a : ORIGIN = 0, LENGTH = 1 } */\n", "notes.txt is not a map file memmap can read", None), + "empty.map": ("", "empty.map is not a map file memmap can read", None), + "fw.elf": (make_elf([code(".text", 0, 16)]), "fw.elf is not a map file memmap can read", None), +} + + +@pytest.mark.parametrize(("name", "case"), NOT_MAPS.items(), ids=NOT_MAPS) +def test_files_that_are_not_maps(tmp_path, name, case): + content, message, hint = case + file = tmp_path / name + file.write_bytes(content if isinstance(content, bytes) else content.encode()) + with pytest.raises(ImportProblem) as raised: + read_map(file) + assert str(raised.value) == message + assert raised.value.hint == (hint or "pass the map file of a GNU ld, armlink or IAR link") + + +def test_unreadable_maps(tmp_path): + with pytest.raises(ImportProblem, match="cannot read .*missing.map"): + read_map(tmp_path / "missing.map") + with pytest.raises(ImportProblem, match="cannot read"): + read_map(tmp_path) # a directory + + +# Resolving regions + + +def linker_file(*regions, kind="GNU ld map"): + return LinkerFile("fw.map", kind, list(regions)) + + +def rows(resolved): + return [(m.id, m.start, m.size, m.source) for m in resolved.memories] + + +def w112(resolved): + """(subject, hint) of each W112 of a resolve.""" + assert {d.code for d in resolved.diagnostics} <= {"W112"} + return [(d.subject, d.hint) for d in resolved.diagnostics] + + +def test_resolve_ids_names_and_sources(): + linker = linker_file(LinkerRegion("FLASH", 0x0, 0x1000), LinkerRegion("RAM_D1", 0x2000, 0x100)) + resolved = resolve_memories(linker, [], 32) + assert [(m.id, m.name, m.source, m.optional) for m in resolved.memories] == [ + ("flash", "FLASH", "fw.map", True), + ("ram_d1", "RAM_D1", "fw.map", True), + ] + assert (resolved.notes, resolved.diagnostics) == ([], []) + + +def test_resolve_leaves_out_what_cannot_be_drawn(): + linker = linker_file( + LinkerRegion("FLASH", 0x0, 0x1000), + LinkerRegion("TOP", 0xFFFFF800, 0x1000), + LinkerRegion("ER_RW", 0x20000000, None), + LinkerRegion("EMPTY", 0x30000000, 0), + ) + resolved = resolve_memories(linker, [], 32) + assert [m.id for m in resolved.memories] == ["flash"] + assert resolved.notes == [ + "Left out of fw.map: TOP (0xFFFFF800, 4K): ends past the 32-bit address space.", + "Left out of fw.map: ER_RW (0x20000000): no size.", + "Left out of fw.map: EMPTY (0x30000000, 0): size 0.", + ] + assert w112(resolved) == [ # a region of size 0 holds nothing, so it loses nothing + ("top", "check that the map belongs to this ELF, or give it with --region top=0xFFFFF800:SIZE"), + ("er_rw", "give it with --region er_rw=0x20000000:SIZE"), + ] + assert resolved.diagnostics[1].message == "ER_RW (0x20000000) in fw.map is left out: no size" + + +@pytest.mark.parametrize( + ("bits", "start", "size", "drawn"), + [ + (32, 0xFFFFF7FF, 0x800, True), # Zephyr's IDT_LIST, ending one byte below the top + (32, 0xFFFFF000, 0x1000, True), # ending on the last address + (32, 0xFFFFF001, 0x1000, False), # one byte past it + (64, 0xFFFFFFFF_FFFFF000, 0x1000, True), + (64, 0xFFFFFFFF_FFFFF001, 0x1000, False), + ], +) +def test_resolve_address_space_edges(bits, start, size, drawn): + linker = linker_file(LinkerRegion("TOP", start, size), LinkerRegion("LOW", 0, 0x100)) + assert [m.id for m in resolve_memories(linker, [], bits).memories] == (["top", "low"] if drawn else ["low"]) + + +def test_resolve_with_region_overrides(): + linker = linker_file(LinkerRegion("FLASH", 0x0, 0x1000), LinkerRegion("ER_RW", 0x4000, None)) + specs = [parse_region_spec("flash=0x0:2K:Flash"), parse_region_spec("er_rw=0x4000:1K"), + parse_region_spec("extra=0x8000:1K")] + resolved = resolve_memories(linker, specs, 32) + assert rows(resolved) == [ + ("flash", 0, 2048, None), + ("er_rw", 0x4000, 1024, None), + ("extra", 0x8000, 1024, None), + ] + assert (resolved.notes, resolved.diagnostics) == ([], []) + + +def test_resolve_repeated_names_get_numbered_ids(): + linker = linker_file(LinkerRegion("RAM", 0, 16), LinkerRegion("ram", 16, 16)) + assert [m.id for m in resolve_memories(linker, [], 32).memories] == ["ram", "ram_2"] + + +def test_resolve_overlapping_map_regions_keep_the_lower_or_larger(): + """GNU ld lets MEMORY regions overlap, as when a script declares one RAM twice, once for + each bus. The import draws flat regions, so each overlapping one but the first is left out, + with W112 only when part of it is lost.""" + linker = linker_file( + LinkerRegion("BOOT", 0x08000000, 0x2000), # inside FLASH, though listed first + LinkerRegion("FLASH", 0x08000000, 0x10000), + LinkerRegion("RAM_IBUS", 0x20000000, 0x5000), + LinkerRegion("RAM_DBUS", 0x20000000, 0x5000), # the same range: the first one stays + LinkerRegion("TAIL", 0x20004000, 0x2000), # starts inside RAM_IBUS, ends past it + ) + resolved = resolve_memories(linker, [], 32) + assert [m.id for m in resolved.memories] == ["flash", "ram_ibus"] + assert resolved.notes == [ + "Left out of fw.map: BOOT (0x08000000, 8K): overlaps FLASH.", + "Left out of fw.map: RAM_DBUS (0x20000000, 20K): overlaps RAM_IBUS.", + "Left out of fw.map: TAIL (0x20004000, 8K): overlaps RAM_IBUS.", + ] + assert w112(resolved) == [ + ("tail", "to draw it instead of RAM_IBUS, give it with --region ram_ibus=0x20004000:8K:TAIL"), + ] + + +def test_resolve_following_the_overlap_hint_draws_the_other_region(): + linker = linker_file(LinkerRegion("RAM", 0x20000000, 0x1000), LinkerRegion("RAM_ALIAS", 0x20000800, 0x10000)) + (hint,) = [d.hint for d in resolve_memories(linker, [], 32).diagnostics] + spec = re.search(r"--region (\S+)", hint).group(1) + resolved = resolve_memories(linker, [parse_region_spec(spec)], 32) + assert rows(resolved) == [("ram", 0x20000800, 0x10000, None)] + assert resolved.diagnostics == [] # RAM_ALIAS lies inside the --region that took its range + + +def test_resolve_region_spec_inside_a_map_region(): + """A new --region inside a map region leaves the map region out. Giving one of the specs + the map region's id splits it instead, without a warning.""" + linker = linker_file(LinkerRegion("FLASH", 0x08000000, 0x10000), LinkerRegion("RAM", 0x20000000, 0x5000)) + specs = [parse_region_spec("boot=0x08000000:8K"), parse_region_spec("app=0x08002000:56K")] + resolved = resolve_memories(linker, specs, 32) + assert [m.id for m in resolved.memories] == ["ram", "boot", "app"] + assert resolved.notes == ["Left out of fw.map: FLASH (0x08000000, 64K): overlaps --region boot."] + assert w112(resolved) == [("flash", "to split FLASH, give one of the --region specs its id, flash")] + specs = [parse_region_spec("flash=0x08000000:8K:Boot"), parse_region_spec("app=0x08002000:56K")] + resolved = resolve_memories(linker, specs, 32) + assert [m.id for m in resolved.memories] == ["flash", "ram", "app"] + assert (resolved.notes, resolved.diagnostics) == ([], []) + + +def test_resolve_region_spec_around_a_map_region(): + linker = linker_file(LinkerRegion("FLASH", 0x08000000, 0x10000)) + resolved = resolve_memories(linker, [parse_region_spec("rom=0x08000000:1M")], 32) + assert [m.id for m in resolved.memories] == ["rom"] + assert resolved.notes == ["Left out of fw.map: FLASH (0x08000000, 64K): overlaps --region rom."] + assert resolved.diagnostics == [] # FLASH lies inside it + + +@pytest.mark.parametrize( + ("specs", "message"), + [ + (["flash=0x08000000:64K", "flash=0x08000000:32K"], "--region flash is given twice"), + (["a=0x08000000:64K", "b=0x08001000:4K"], "--region a and --region b overlap"), + ], +) +def test_resolve_rejects_bad_region_specs(specs, message): + linker = linker_file(LinkerRegion("FLASH", 0x08000000, 0x10000)) + with pytest.raises(ImportProblem, match=re.escape(message)): + resolve_memories(linker, [parse_region_spec(spec) for spec in specs], 32) + + +GIVE_EACH = "give each region with --region ID=START:SIZE" +NOTHING_TO_DRAW = { + "lld": ( + linker_file(kind="lld map"), + "fw.map is an lld map, which lists no memory regions", + GIVE_EACH + ", copying ORIGIN and LENGTH from the linker script's MEMORY block", + ), + "gnu without MEMORY": ( + linker_file(), + "fw.map lists no memory regions, as its linker script has no MEMORY block", + GIVE_EACH, + ), + "iar without ranges": (linker_file(kind="IAR map"), "fw.map lists no memory regions", GIVE_EACH), + "no sizes": ( + linker_file(LinkerRegion("ER_RO", 0x08000000, None), LinkerRegion("ER_RW", 0x20000000, None), + LinkerRegion("ER_ZI", 0x20000000, None), kind="armlink map"), + ("fw.map has no region memmap can draw (ER_RO (0x08000000): no size; ER_RW (0x20000000): " + "no size; ER_ZI (0x20000000): no size)"), + "give them with --region, such as --region er_ro=0x08000000:SIZE --region er_rw=0x20000000:SIZE", + ), + "none that fit": ( + linker_file(LinkerRegion("TOP", 0xFFFFF800, 0x1000), LinkerRegion("NONE", 0x0, 0)), + ("fw.map has no region memmap can draw (TOP (0xFFFFF800, 4K): ends past the 32-bit " + "address space; NONE (0x00000000, 0): size 0)"), + GIVE_EACH, + ), +} + + +@pytest.mark.parametrize(("linker", "message", "hint"), NOTHING_TO_DRAW.values(), ids=NOTHING_TO_DRAW) +def test_resolve_with_nothing_to_draw(linker, message, hint): + with pytest.raises(ImportProblem) as raised: + resolve_memories(linker, [], 32) + assert (str(raised.value), raised.value.hint) == (message, hint) + + +def test_resolve_map_without_regions_uses_the_specs(): + specs = [parse_region_spec("flash=0x0:1M"), parse_region_spec("ram=0x20000000:256K")] + resolved = resolve_memories(linker_file(kind="lld map"), specs, 32) + assert rows(resolved) == [("flash", 0, 1 << 20, None), ("ram", 0x20000000, 256 << 10, None)] + assert (resolved.notes, resolved.diagnostics) == (["fw.map is an lld map, which lists no memory regions."], []) + + +def test_resolve_one_w112_per_start_of_regions_without_a_size(): + linker = linker_file(LinkerRegion("ER_RO", 0x08000000, 0x10000), LinkerRegion("ER_RW", 0x20000000, None), + LinkerRegion("ER_ZI", 0x20000000, None)) + resolved = resolve_memories(linker, [], 32) + assert w112(resolved) == [("er_rw", "give it with --region er_rw=0x20000000:SIZE")] + assert len(resolved.notes) == 2 # both are noted + resolved = resolve_memories(linker, [parse_region_spec("er_rw=0x20000000:20K")], 32) + assert resolved.notes == ["Left out of fw.map: ER_ZI (0x20000000): no size."] + assert resolved.diagnostics == [] # its start lies in er_rw, which is drawn + + +def test_resolve_notes_that_iar_regions_are_placement_ranges(): + resolved = resolve_memories(read_map(SNAPSHOTS / "iar8_stm32f030.map"), [], 32) + assert resolved.notes == [ + ("iar8_stm32f030.map is an IAR map, which names no regions: these are its placement ranges, " + "called ROM, RAM or Memory by what is placed there.") + ] + + +def test_resolve_iar_ids_follow_address_order(): + """CCM RAM at 0x10000000 comes first, so the main SRAM is ram_2, and that is the id that + replaces it.""" + linker = read_map(SNAPSHOTS / "iar8_stm32f407_ccm.map") + assert [m.id for m in resolve_memories(linker, [], 32).memories] == ["rom", "ram", "ram_2"] + resolved = resolve_memories(linker, [parse_region_spec("ram_2=0x20000000:128K:SRAM")], 32) + assert [(m.id, m.name) for m in resolved.memories] == [("rom", "ROM"), ("ram", "RAM"), ("ram_2", "SRAM")] + + +# Importing with a map + + +def test_cli_map_matches_region_specs(tmp_path): + by_region = tmp_path / "region.yaml" + by_map = tmp_path / "map.yaml" + assert invoke("from-elf", FIRMWARE, "-o", by_region, "--region", "flash=0x08000000:64K", + "--region", "ram=0x20000000:20K").exit_code == 0 + assert invoke("from-elf", FIRMWARE, "-o", by_map, "--map", FIRMWARE_MAP, "--strict").exit_code == 0 + body = [file.read_text().split("\n", 2)[2] for file in (by_region, by_map)] + assert body[0] == body[1] + assert "memory regions from firmware.map. Rename" in header(by_map) + + +def test_cli_64_bit_image_with_a_map(tmp_path): + out = tmp_path / "out.yaml" + result = invoke("from-elf", FIXTURES / "firmware64.elf", "-o", out, "--map", FIRMWARE_MAP, "--strict") + assert result.exit_code == 0, result.output + assert " start: 0x0000000008000000\n size: 64K" in out.read_text() + + +def test_regions_with_nothing_in_them_get_no_column(tmp_path): + """The STM32F407 Keil map has a CCM RAM region that this image does not use.""" + elf = cortex_m(tmp_path) + result = invoke("from-elf", elf, "--map", SNAPSHOTS / "keil_f407_compressed.map", "--strict") + assert result.exit_code == 0, result.output + text = (tmp_path / "fw.yaml").read_text() + assert '- id: "rw_iram2"' in text # still in memory + assert 'root: "rw_iram2"' not in text + assert "# No section lies in RW_IRAM2. To draw one, add a column with its id as root." in text + + +def test_cli_zephyr_map(tmp_path): + """Zephyr declares IDT_LIST just below the top of memory for a build step of its own. It + stays without a column, and so does not stretch the drawing.""" + elf = elf_file(tmp_path, [code(".text", 0x0, 0x1000), code(".data", 0x20000000, 0x100)]) + result = invoke("from-elf", elf, "--map", SNAPSHOTS / "zephyr_nrf52dk.map", "--strict") + assert result.exit_code == 0, result.output + text = (tmp_path / "fw.yaml").read_text() + assert " # No section lies in IDT_LIST. To draw one" in text + assert [line.split(",")[1] for line in text.splitlines() if "root:" in line] == [ + ' root: "flash"', + ' root: "sram"', + ] + + +def test_cli_reports_left_out_regions(tmp_path): + """A region left out is noted in the config and reported as W112, which --strict fails on.""" + top = "TOP 0xfffff800 0x00001000" + map_file = write(tmp_path, "fw.map", GNU_MAP.replace("a_very_long_region_name 0x30000000 0x00000100", top)) + elf = elf_file(tmp_path, [code(".text", 0x08000000, 0x1000)]) + result = invoke("from-elf", elf, "--map", map_file, "--json") + assert result.exit_code == 0, result.output + assert warnings(result) == [ + ("W112", "top", "check that the map belongs to this ELF, or give it with --region top=0xFFFFF800:SIZE") + ] + assert "\n# Left out of fw.map: TOP (0xFFFFF800, 4K): ends past the 32-bit address space." in ( + header(tmp_path / "fw.yaml") + ) + assert '- id: "top"' not in (tmp_path / "fw.yaml").read_text() + strict = invoke("from-elf", elf, "--map", map_file, "--force", "--strict") + assert strict.exit_code == 1 + assert "W112" in strict.output and "TOP (0xFFFFF800, 4K) in fw.map is left out" in strict.output + + +def test_cli_region_replaces_a_map_region(tmp_path): + out = tmp_path / "out.yaml" + result = invoke("from-elf", FIRMWARE, "-o", out, "--map", FIRMWARE_MAP, "--region", "flash=0x08000000:128K:Flash") + assert result.exit_code == 0, result.output + assert "memory regions from firmware.map and --region." in header(out) + assert 'name: "Flash"\n start: 0x08000000\n size: 128K' in out.read_text() + + +def test_cli_regions_all_replaced_come_from_region(tmp_path): + out = tmp_path / "out.yaml" + result = invoke("from-elf", FIRMWARE, "-o", out, "--map", FIRMWARE_MAP, "--region", "flash=0x08000000:64K", + "--region", "ram=0x20000000:20K", "--strict") + assert result.exit_code == 0, result.output + assert "memory regions from --region. Rename" in header(out) + + +def test_cli_overlapping_regions_in_a_map(tmp_path): + """One RAM declared twice loses nothing, so --strict passes.""" + map_file = write(tmp_path, "fw.map", gnu_map( + "FLASH 0x08000000 0x00010000 xr", "RAM 0x20000000 0x00005000 xrw", "RAM_DBUS 0x20000000 0x00005000 rw")) + result = invoke("from-elf", cortex_m(tmp_path), "--map", map_file, "--strict") + assert result.exit_code == 0, result.output + assert "# Left out of fw.map: RAM_DBUS (0x20000000, 20K): overlaps RAM." in header(tmp_path / "fw.yaml") + + +def test_cli_section_crossing_a_map_region(tmp_path): + """A map region too small for a section is left out, and the import goes on.""" + map_file = write(tmp_path, "small.map", gnu_map("FLASH 0x08000000 0x00000100 xr", "RAM 0x20000000 0x5000 xrw")) + result = invoke("from-elf", cortex_m(tmp_path), "--map", map_file, "--json") + assert result.exit_code == 0, result.output + assert warnings(result) == [ + ("W112", "flash", "give it with --region flash=0x08000000:SIZE, large enough to hold .text"), + ("W110", "text", "check that the map belongs to this ELF, or add a --region that covers it"), + ] + note = "# Left out of small.map: FLASH (0x08000000, 256): .text (0x08000000 - 0x08000FFF) runs across its edge." + assert note in header(tmp_path / "fw.yaml") + + +def test_cli_armlink_load_image_past_its_execution_region(tmp_path): + """armlink places the RW load image after the RO code, inside the load region but past a + tight execution region Max.""" + text = ( + "Memory Map of the image\n" + " Load Region LR_IROM1 (Base: 0x08000000, Size: 0x00001100, Max: 0x00080000, ABSOLUTE)\n" + " Execution Region ER_IROM1 (Exec base: 0x08000000, Load base: 0x08000000, Size: " + "0x00000ff0, Max: 0x00001000, ABSOLUTE)\n" + " Execution Region RW_IRAM1 (Exec base: 0x20000000, Load base: 0x08000ff0, Size: " + "0x00000100, Max: 0x00005000, ABSOLUTE)\n" + ) + map_file = write(tmp_path, "keil.map", text) + elf = tmp_path / "fw.elf" + sections = [code(".text", 0x08000000, 0xFF0), code(".data", 0x20000000, 0x100, offset=0x2000)] + elf.write_bytes(make_elf(sections, [(0x2000, 0x20000000, 0x08000FF0, 0x100, 0x100)])) # .data loads at 0x08000FF0 + result = invoke("from-elf", elf, "--map", map_file, "--json") + assert result.exit_code == 0, result.output + assert [(code_, subject) for code_, subject, _ in warnings(result)] == [ + ("W112", "er_irom1"), ("W110", "text"), ("W110", "data_load"), + ] + + +def test_cli_section_crossing_a_region_spec_still_stops(tmp_path): + result = invoke("from-elf", cortex_m(tmp_path), "--region", "flash=0x08000000:256", "--json") + message, hint = usage_problem(result) + assert "crosses the edge of --region flash" in message and hint == "check its start and size" + + +def test_cli_map_of_another_image(tmp_path): + """Every section outside the map's regions points to the wrong map.""" + elf = elf_file(tmp_path, [code(".text", 0x0, 0x1000), code(".data", 0x10000000, 0x100)]) + result = invoke("from-elf", elf, "--map", FIRMWARE_MAP, "--json") + hint = "check that the map belongs to this ELF, or add a --region that covers it" + assert warnings(result) == [("W110", "text", hint), ("W110", "data", hint)] + assert invoke("from-elf", elf, "--map", FIRMWARE_MAP, "--force", "--strict").exit_code == 1 + + +def test_sections_outside_the_iar_regions(tmp_path): + """IAR lists only the ranges it placed content in, so a section placed at a fixed address + outside them, such as an i.MX RT boot header, is outside every region.""" + elf = elf_file(tmp_path, [code(".flash_conf", 0x08000400, 0x200), code(".text", 0x08001000, 0x2000), + code(".data", 0x20080000, 0x100)]) + result = invoke("from-elf", elf, "--map", SNAPSHOTS / "iar8_rt685.map", "--json") + assert [(code_, subject) for code_, subject, _ in warnings(result)] == [("W110", "flash_conf")] + assert "# iar8_rt685.map is an IAR map, which names no regions" in header(tmp_path / "fw.yaml") + + +# Maps without enough to go on + + +def test_cli_lld_map_needs_region(tmp_path): + map_file = write(tmp_path, "fw.map", LLD_HEADER) + out = tmp_path / "out.yaml" + message, hint = usage_problem(invoke("from-elf", cortex_m(tmp_path), "-o", out, "--map", map_file, "--json")) + assert message == "fw.map is an lld map, which lists no memory regions" + assert hint.startswith("give each region with --region ID=START:SIZE, copying ORIGIN and LENGTH") + assert not out.exists() + text = invoke("from-elf", tmp_path / "fw.elf", "-o", out, "--map", map_file) + assert text.exit_code == 2 + assert text.output == f"memmap: {message}\n hint: {hint}\n" + result = invoke("from-elf", tmp_path / "fw.elf", "-o", out, "--map", map_file, "--region", "flash=0x08000000:64K", + "--region", "ram=0x20000000:20K", "--strict") + assert result.exit_code == 0, result.output + assert header(out).endswith( + "memory regions from --region. Rename, merge or annotate blocks freely.\n" + "# fw.map is an lld map, which lists no memory regions." + ) + + +def test_cli_map_without_memory_needs_region(tmp_path): + map_file = write(tmp_path, "fw.map", gnu_map()) + message, hint = usage_problem(invoke("from-elf", cortex_m(tmp_path), "--map", map_file, "--json")) + assert message.endswith("as its linker script has no MEMORY block") and hint == GIVE_EACH + + +def test_cli_map_without_sizes_suggests_the_regions_to_give(tmp_path): + """Keil without a scatter file gives each region a start but no size. The suggested + --region specs, with sizes filled in, are what it takes, and pass --strict.""" + elf = cortex_m(tmp_path) + keil = SNAPSHOTS / "keil_no_scatter.map" + _, hint = usage_problem(invoke("from-elf", elf, "--map", keil, "--json")) + suggested = re.findall(r"--region (\S+):SIZE", hint) + assert suggested == ["er_ro=0x08000000", "er_rw=0x20000000"] + specs = [arg for spec, size in zip(suggested, ("64K", "20K")) for arg in ("--region", f"{spec}:{size}")] + result = invoke("from-elf", elf, "--map", keil, *specs, "--strict") + assert result.exit_code == 0, result.output + assert "# Left out of keil_no_scatter.map: ER_ZI (0x20000000): no size." in header(tmp_path / "fw.yaml") + + +def test_cli_map_missing_one_size_names_it(tmp_path): + """With flash given and RAM still without a size, W112 says which --region is missing.""" + result = invoke("from-elf", cortex_m(tmp_path), "--map", SNAPSHOTS / "keil_no_scatter.map", + "--region", "er_ro=0x08000000:64K", "--json") + assert warnings(result) == [ + ("W112", "er_rw", "give it with --region er_rw=0x20000000:SIZE"), + ("W110", "data", "add a --region that covers it, or check the start and size of yours"), + ] + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--map", "missing.map"], "cannot read missing.map"), + (["--map", "tests"], "cannot read tests"), + (["--map", "tests/fixtures/elf_src/firmware.ld"], "firmware.ld is a linker script"), + (["--map", "tests/fixtures/firmware.map", "--region", "flash=0x08000000:64K", "--region", + "flash=0x08000000:32K"], "--region flash is given twice"), + (["--map", "tests/fixtures/firmware.map", "--region", "a=0x0:1K", "--region", "b=0x100:1K"], + "--region a and --region b overlap"), + ], +) +def test_cli_map_usage_problems(tmp_path, monkeypatch, args, message): + monkeypatch.chdir(ROOT) + result = invoke("from-elf", FIRMWARE, "-o", tmp_path / "out.yaml", *args, "--json") + assert message in usage_problem(result)[0] + assert not (tmp_path / "out.yaml").exists() + + +def test_cli_bad_iar_number_is_a_usage_problem(tmp_path): + map_file = write(tmp_path, "fw.map", "# IAR ELF Linker\n*** PLACEMENT SUMMARY\n\"P1\": place in [from 0x' to 0x1] { rw };\n") + message, _ = usage_problem(invoke("from-elf", FIRMWARE, "-o", tmp_path / "out.yaml", "--map", map_file, "--json")) + assert message == "cannot read the address \"0x'\" in the placement summary" + + +def test_cli_crash_still_prints_one_json_object(tmp_path, monkeypatch): + def broken(path): + raise RuntimeError("boom") + + monkeypatch.setattr("memmap.cli.read_map", broken) + result = invoke("from-elf", FIRMWARE, "-o", tmp_path / "out.yaml", "--map", FIRMWARE_MAP, "--json") + assert result.exit_code == 3 + (diagnostic,) = json.loads(result.stdout)["diagnostics"] + assert (diagnostic["code"], diagnostic["message"]) == ("E091", "internal error: boom")