From ce0ef425c59907ca79bba0146307f3897b9f4f5c Mon Sep 17 00:00:00 2001 From: Nikolay Malkovsky Date: Mon, 21 Sep 2026 21:41:01 +0300 Subject: [PATCH] Permutations and permutable sequences --- AGENTS.md | 17 +- CMakeLists.txt | 33 + README.md | 2 + academic/AGENTS.md | 4 + academic/bibliography/references.bib | 693 +++++++- .../compact-integer-sets-vectors/.gitignore | 2 + agentic/cpp | 2 +- include/pixie/bits.h | 102 +- include/pixie/detail/sequence/bit_block.h | 205 +++ .../pixie/detail/sequence/packed_bit_block.h | 27 + .../detail/sequence/packed_value_block.h | 178 ++ include/pixie/detail/sequence/pointer_block.h | 165 ++ include/pixie/detail/sequence/sequence_tree.h | 1338 +++++++++++++++ .../detail/sequence/sequence_tree_kernels.h | 94 ++ .../pixie/experimental/permuted_bit_block.h | 284 ++++ include/pixie/permutable_sequence.h | 148 ++ include/pixie/permutation.h | 129 ++ include/pixie/permutations/permutation.h | 246 +++ .../permutation_implementations.h | 58 + include/pixie/permutations/sequence.h | 542 ++++++ .../permutations/sequence_implementations.h | 65 + include/pixie/sequence_options.h | 15 + src/benchmarks/bit_sequence_benchmarks.cpp | 854 ++++++++++ src/benchmarks/packed_copy_kernels.h | 132 ++ .../permutable_sequence_benchmarks.cpp | 56 + src/benchmarks/permutation_benchmarks.cpp | 44 + .../sequence_controls_benchmarks.cpp | 24 + src/benchmarks/sequence_facade_benchmarks.h | 632 +++++++ .../sequence_tree_kernels_benchmarks.cpp | 97 ++ src/tests/bit_sequence_tests.cpp | 1463 +++++++++++++++++ src/tests/bits_unittests.cc | 74 + src/tests/packed_sequence_tests.cpp | 413 +++++ src/tests/permutable_sequence_tests.cpp | 930 +++++++++++ src/tests/permutation_tests.cpp | 522 ++++++ src/tests/sequence_tree_kernels_tests.cpp | 137 ++ 35 files changed, 9719 insertions(+), 8 deletions(-) create mode 100644 academic/notes/compact-integer-sets-vectors/.gitignore create mode 100644 include/pixie/detail/sequence/bit_block.h create mode 100644 include/pixie/detail/sequence/packed_bit_block.h create mode 100644 include/pixie/detail/sequence/packed_value_block.h create mode 100644 include/pixie/detail/sequence/pointer_block.h create mode 100644 include/pixie/detail/sequence/sequence_tree.h create mode 100644 include/pixie/detail/sequence/sequence_tree_kernels.h create mode 100644 include/pixie/experimental/permuted_bit_block.h create mode 100644 include/pixie/permutable_sequence.h create mode 100644 include/pixie/permutation.h create mode 100644 include/pixie/permutations/permutation.h create mode 100644 include/pixie/permutations/permutation_implementations.h create mode 100644 include/pixie/permutations/sequence.h create mode 100644 include/pixie/permutations/sequence_implementations.h create mode 100644 include/pixie/sequence_options.h create mode 100644 src/benchmarks/bit_sequence_benchmarks.cpp create mode 100644 src/benchmarks/packed_copy_kernels.h create mode 100644 src/benchmarks/permutable_sequence_benchmarks.cpp create mode 100644 src/benchmarks/permutation_benchmarks.cpp create mode 100644 src/benchmarks/sequence_controls_benchmarks.cpp create mode 100644 src/benchmarks/sequence_facade_benchmarks.h create mode 100644 src/benchmarks/sequence_tree_kernels_benchmarks.cpp create mode 100644 src/tests/bit_sequence_tests.cpp create mode 100644 src/tests/packed_sequence_tests.cpp create mode 100644 src/tests/permutable_sequence_tests.cpp create mode 100644 src/tests/permutation_tests.cpp create mode 100644 src/tests/sequence_tree_kernels_tests.cpp diff --git a/AGENTS.md b/AGENTS.md index d91f533..cf4f514 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,8 @@ Current library families are: - rank/select support over packed bit sequences; - positional and monotone packed integer vectors; +- owning permutations with rebased consuming merge and permutable sequences with + unchanged-value consuming merge; - range min-max (RmM) indexes; - static range-minimum-query (RMQ) indexes; - rooted-tree encodings (LOUDS, balanced parentheses, and DFUDS); @@ -73,7 +75,14 @@ adds Pixie context; it does not replace the shared guidance. Public data-structure families use CRTP contracts. The current contracts are `IntegerVectorBase`, `MonotoneIntegerVectorBase`, `RankSelectBase`, `RmMBase`, -`pixie::rmq::RmqBase`, `TreeBase`, `StorageBase`, and `WaveletTreeBase`. +`pixie::rmq::RmqBase`, `TreeBase`, `StorageBase`, `WaveletTreeBase`, +`PermutationBase`, and `PermutableSequenceBase`. The latter two keep separate +contracts in `permutation.h` and `permutable_sequence.h`, with concrete +`Permutation` and `PermutableSequence` types in `permutations/permutation.h` and +`permutations/sequence.h`. Their benchmark catalogs are +`permutations/permutation_implementations.h` and +`permutations/sequence_implementations.h`, respectively; +`detail/sequence/` is their unsupported shared engine, not a public family. 1. Define or extend the public contract in `include/pixie/.h`. Public facade methods delegate to a clearly named `*_impl()` method on the @@ -248,8 +257,10 @@ The registered test executables are `bit_algorithms_unittests`, `rank_select_unittests`, `rank_select_tests`, `benchmark_tests`, `test_rmm`, `tree_tests`, `wavelet_tree_tests`, `storage_tests`, `serialization_tests`, `integer_vector_tests`, `excess_positions_tests`, -`excess_record_lows_tests`, and `rmq_tests`. Run an executable directly only -when debugging a focused Google Test filter. +`excess_record_lows_tests`, `rmq_tests`, `permutation_tests`, +`permutable_sequence_tests`, and `bit_sequence_tests`. The last covers internal +sequence blocks/tree/kernels and the retained mapped-block experiment. Run an +executable directly only when debugging a focused Google Test filter. ### Test Configuration via Environment Variables diff --git a/CMakeLists.txt b/CMakeLists.txt index f6aa200..4743f48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -309,7 +309,25 @@ if (PIXIE_TESTS) PRIVATE ${sdsl_lite_SOURCE_DIR}/include) endif () + add_executable(bit_sequence_tests + src/tests/bit_sequence_tests.cpp + src/tests/packed_sequence_tests.cpp + src/tests/sequence_tree_kernels_tests.cpp) + target_link_libraries(bit_sequence_tests + pixie::pixie gtest_main) + target_compile_definitions(bit_sequence_tests + PRIVATE PIXIE_SEQUENCE_TREE_TESTING) + + foreach (family IN ITEMS permutation permutable_sequence) + add_executable(${family}_tests src/tests/${family}_tests.cpp) + target_link_libraries(${family}_tests PRIVATE pixie::pixie gtest_main) + target_compile_definitions(${family}_tests PRIVATE PIXIE_SEQUENCE_TREE_TESTING) + endforeach () + set(PIXIE_TEST_TARGETS + permutation_tests + permutable_sequence_tests + bit_sequence_tests bit_algorithms_unittests rank_select_unittests rank_select_tests @@ -493,7 +511,22 @@ if (PIXIE_BENCHMARKS) benchmark_main ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(bit_sequence_benchmarks + src/benchmarks/bit_sequence_benchmarks.cpp + src/benchmarks/sequence_controls_benchmarks.cpp + src/benchmarks/sequence_tree_kernels_benchmarks.cpp) + target_link_libraries(bit_sequence_benchmarks + pixie::pixie benchmark benchmark_main) + + foreach (family IN ITEMS permutation permutable_sequence) + add_executable(${family}_benchmarks src/benchmarks/${family}_benchmarks.cpp) + target_link_libraries(${family}_benchmarks PRIVATE pixie::pixie benchmark benchmark_main) + endforeach () + set(PIXIE_BENCHMARK_TARGETS + permutation_benchmarks + permutable_sequence_benchmarks + bit_sequence_benchmarks rank_select_benchmarks rmm_benchmarks rmm_btree_benchmarks diff --git a/README.md b/README.md index 6a7b622..a94bc50 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ - **Succinct trees**: static variants of $2$-bit per entry trees, i.e. LOUDS, DFUDS, BP (Based on Euler tour and Ferrada-Navarro style). - **Wavelet tree**, i.e. static structure that supposts rank/select on arbitrary finite alphabets, supports building a Huffman archieve with fast extraction of arbitrary segment. - Succinct **cartesian tree** and a state of the art solution to static **RMQ** (array is immutable, queries are not known in advance). +- **Permutations**: identity construction, checked reads, rotations, and consuming rebased merge. Contract: `pixie/permutation.h`; concrete: `pixie/permutations/permutation.h`. +- **Permutable sequences**: consuming range construction, checked immutable reads, rotations, and unchanged-value merge. Contract: `pixie/permutable_sequence.h`; concrete: `pixie/permutations/sequence.h`. --- diff --git a/academic/AGENTS.md b/academic/AGENTS.md index 5d1a407..4c733ed 100644 --- a/academic/AGENTS.md +++ b/academic/AGENTS.md @@ -75,6 +75,10 @@ quarto render index.qmd --to pdf quarto preview index.qmd ``` +RevealJS loads SVG figures through ``, where nested external SVG references +such as `` may disappear even though PNG exporters resolve +them. Keep presentation SVGs standalone by inlining all required layers. + ## Conventions ### Paper Template (ACM Format) diff --git a/academic/bibliography/references.bib b/academic/bibliography/references.bib index b217ae3..6a1bf38 100644 --- a/academic/bibliography/references.bib +++ b/academic/bibliography/references.bib @@ -122,7 +122,7 @@ @misc{sdsl-lite-v3 author = {{sdsl-lite contributors}}, title = {sdsl-lite v3: Succinct Data Structure Library}, year = {2024}, - url = {https://github.com/sdsl-lite/sdsl-lite}, + url = {https://github.com/xxsds/sdsl-lite}, note = {GitHub repository} } @@ -179,10 +179,71 @@ @inproceedings{bender2000 title = {The {LCA} Problem Revisited}, booktitle = {Latin American Theoretical Informatics Symposium (LATIN)}, pages = {88--94}, + publisher = {Springer}, + address = {Berlin, Heidelberg}, year = {2000}, doi = {10.1007/10719839_9} } +@techreport{AlonSchieber1987, + author = {Alon, Noga and Schieber, Baruch}, + title = {Optimal Preprocessing for Answering On-Line Product Queries}, + institution = {The Moise and Frida Eskenasy Institute of Computer Science, Tel Aviv University}, + number = {71/87}, + year = {1987}, + eprint = {2406.06321}, + archivePrefix = {arXiv}, + primaryClass = {cs.DS}, + note = {Author-uploaded archival copy published in 2024}, + url = {https://arxiv.org/abs/2406.06321} +} + +@inproceedings{BerkmanEtAl1989, + author = {Berkman, Omer and Breslauer, Dany and Galil, Zvi and Schieber, Baruch and Vishkin, Uzi}, + title = {Highly Parallelizable Problems}, + booktitle = {Proceedings of the 21st Annual ACM Symposium on Theory of Computing (STOC)}, + pages = {309--319}, + publisher = {ACM}, + address = {New York, NY, USA}, + year = {1989}, + doi = {10.1145/73007.73036}, + url = {https://doi.org/10.1145/73007.73036} +} + +@techreport{BerkmanVishkin1990, + author = {Berkman, Omer and Vishkin, Uzi}, + title = {Recursive Star-Tree Parallel Data-Structure}, + institution = {University of Maryland Institute for Advanced Computer Studies}, + number = {UMIACS-TR-90-40 and CS-TR-2437}, + year = {1990}, + doi = {10.21236/ADA227803}, + url = {https://archive.org/details/DTIC_ADA227803} +} + +@inproceedings{Yao1982, + author = {Yao, Andrew C.}, + title = {Space-Time Tradeoff for Answering Range Queries (Extended Abstract)}, + booktitle = {Proceedings of the 14th Annual ACM Symposium on Theory of Computing (STOC)}, + pages = {128--136}, + publisher = {ACM}, + address = {New York, NY, USA}, + year = {1982}, + doi = {10.1145/800070.802185}, + url = {https://doi.org/10.1145/800070.802185} +} + +@article{BenderEtAl2005, + author = {Bender, Michael A. and Farach-Colton, Mart{\'\i}n and Pemmasani, Giridhar and Skiena, Steven and Sumazin, Pavel}, + title = {Lowest Common Ancestors in Trees and Directed Acyclic Graphs}, + journal = {Journal of Algorithms}, + volume = {57}, + number = {2}, + pages = {75--94}, + year = {2005}, + doi = {10.1016/j.jalgor.2005.08.001}, + url = {https://doi.org/10.1016/j.jalgor.2005.08.001} +} + @article{Ferrada2017, author = {Ferrada, H{\'e}ctor and Navarro, Gonzalo}, title = {Improved Range Minimum Queries}, @@ -190,7 +251,56 @@ @article{Ferrada2017 volume = {43}, pages = {72--80}, year = {2017}, - url = {https://dblp.org/rec/journals/jda/FerradaN17} + doi = {10.1016/j.jda.2016.09.002}, + url = {https://doi.org/10.1016/j.jda.2016.09.002} +} + +@article{FischerHeun2011, + author = {Fischer, Johannes and Heun, Volker}, + title = {Space-Efficient Preprocessing Schemes for Range Minimum Queries on Static Arrays}, + journal = {SIAM Journal on Computing}, + volume = {40}, + number = {2}, + pages = {465--492}, + year = {2011}, + doi = {10.1137/090779759}, + url = {https://doi.org/10.1137/090779759} +} + +@misc{Fischer2008, + author = {Fischer, Johannes}, + title = {Optimal Succinctness for Range Minimum Queries}, + year = {2008}, + eprint = {0812.2775}, + archivePrefix = {arXiv}, + primaryClass = {cs.DS}, + url = {https://arxiv.org/abs/0812.2775} +} + +@inproceedings{Baumstark2017, + author = {Baumstark, Niklas and Gog, Simon and Heuer, Tobias and Labeit, Julian}, + title = {Practical Range Minimum Queries Revisited}, + booktitle = {16th International Symposium on Experimental Algorithms (SEA 2017)}, + series = {Leibniz International Proceedings in Informatics (LIPIcs)}, + volume = {75}, + pages = {12:1--12:16}, + publisher = {Schloss Dagstuhl -- Leibniz-Zentrum f{\"u}r Informatik}, + address = {Dagstuhl, Germany}, + year = {2017}, + doi = {10.4230/LIPIcs.SEA.2017.12}, + url = {https://drops.dagstuhl.de/entities/document/10.4230/LIPIcs.SEA.2017.12} +} + +@article{Davoodi2014, + author = {Davoodi, Pooya and Navarro, Gonzalo and Raman, Rajeev and Rao, S. Srinivasa}, + title = {Encoding Range Minima and Range Top-2 Queries}, + journal = {Philosophical Transactions of the Royal Society A: Mathematical, Physical and Engineering Sciences}, + volume = {372}, + number = {2016}, + pages = {20130131}, + year = {2014}, + doi = {10.1098/rsta.2013.0131}, + url = {https://doi.org/10.1098/rsta.2013.0131} } @misc{Kowalski2017FasterRMQ, @@ -203,6 +313,243 @@ @misc{Kowalski2017FasterRMQ url = {https://arxiv.org/abs/1711.10385} } +@article{AhoHopcroftUllman1976, + author = {Aho, Alfred V. and Hopcroft, John E. and Ullman, Jeffrey D.}, + title = {On Finding Lowest Common Ancestors in Trees}, + journal = {SIAM Journal on Computing}, + volume = {5}, + number = {1}, + pages = {115--132}, + year = {1976}, + doi = {10.1137/0205011}, + url = {https://doi.org/10.1137/0205011} +} + +@article{Vuillemin1980, + author = {Vuillemin, Jean}, + title = {A Unifying Look at Data Structures}, + journal = {Communications of the ACM}, + volume = {23}, + number = {4}, + pages = {229--239}, + year = {1980}, + doi = {10.1145/358841.358852}, + url = {https://doi.org/10.1145/358841.358852} +} + +@inproceedings{GabowBentleyTarjan1984, + author = {Gabow, Harold N. and Bentley, Jon Louis and Tarjan, Robert E.}, + title = {Scaling and Related Techniques for Geometry Problems}, + booktitle = {Proceedings of the 16th Annual ACM Symposium on Theory of Computing (STOC)}, + pages = {135--143}, + publisher = {ACM}, + address = {New York, NY, USA}, + year = {1984}, + doi = {10.1145/800057.808675}, + url = {https://doi.org/10.1145/800057.808675} +} + +@article{HarelTarjan1984, + author = {Harel, Dov and Tarjan, Robert E.}, + title = {Fast Algorithms for Finding Nearest Common Ancestors}, + journal = {SIAM Journal on Computing}, + volume = {13}, + number = {2}, + pages = {338--355}, + year = {1984}, + doi = {10.1137/0213024}, + url = {https://doi.org/10.1137/0213024} +} + +@article{SchieberVishkin1988, + author = {Schieber, Baruch and Vishkin, Uzi}, + title = {On Finding Lowest Common Ancestors: Simplification and Parallelization}, + journal = {SIAM Journal on Computing}, + volume = {17}, + number = {6}, + pages = {1253--1262}, + year = {1988}, + doi = {10.1137/0217079}, + url = {https://doi.org/10.1137/0217079} +} + +@article{BerkmanVishkin1993, + author = {Berkman, Omer and Vishkin, Uzi}, + title = {Recursive Star-Tree Parallel Data Structure}, + journal = {SIAM Journal on Computing}, + volume = {22}, + number = {2}, + pages = {221--242}, + year = {1993}, + doi = {10.1137/0222017}, + url = {https://doi.org/10.1137/0222017} +} + +@inproceedings{FischerHeun2006, + author = {Fischer, Johannes and Heun, Volker}, + title = {Theoretical and Practical Improvements on the {RMQ}-Problem, with Applications to {LCA} and {LCE}}, + booktitle = {Combinatorial Pattern Matching (CPM)}, + series = {Lecture Notes in Computer Science}, + volume = {4009}, + pages = {36--48}, + publisher = {Springer}, + address = {Berlin, Heidelberg}, + year = {2006}, + doi = {10.1007/11780441_5}, + url = {https://doi.org/10.1007/11780441_5} +} + +@inproceedings{FischerHeun2007, + author = {Fischer, Johannes and Heun, Volker}, + title = {A New Succinct Representation of {RMQ}-Information and Improvements in the Enhanced Suffix Array}, + booktitle = {Combinatorics, Algorithms, Probabilistic and Experimental Methodologies (ESCAPE)}, + series = {Lecture Notes in Computer Science}, + volume = {4614}, + pages = {459--470}, + publisher = {Springer}, + address = {Berlin, Heidelberg}, + year = {2007}, + doi = {10.1007/978-3-540-74450-4_41}, + url = {https://doi.org/10.1007/978-3-540-74450-4_41} +} + +@inproceedings{Fischer2010, + author = {Fischer, Johannes}, + title = {Optimal Succinctness for Range Minimum Queries}, + booktitle = {LATIN 2010: Theoretical Informatics}, + series = {Lecture Notes in Computer Science}, + volume = {6034}, + pages = {158--169}, + publisher = {Springer}, + address = {Berlin, Heidelberg}, + year = {2010}, + doi = {10.1007/978-3-642-12200-2_16}, + url = {https://doi.org/10.1007/978-3-642-12200-2_16} +} + +@inproceedings{FischerHeunStuhler2008, + author = {Fischer, Johannes and Heun, Volker and St{\"u}hler, Horst Martin}, + title = {Practical Entropy-Bounded Schemes for {$O(1)$}-Range Minimum Queries}, + booktitle = {Proceedings of the Data Compression Conference (DCC)}, + pages = {272--281}, + publisher = {IEEE}, + address = {Los Alamitos, CA, USA}, + year = {2008}, + doi = {10.1109/DCC.2008.45}, + url = {https://doi.org/10.1109/DCC.2008.45} +} + +@inproceedings{DavoodiRamanSatti2012, + author = {Davoodi, Pooya and Raman, Rajeev and Satti, Srinivasa Rao}, + title = {Succinct Representations of Binary Trees for Range Minimum Queries}, + booktitle = {Computing and Combinatorics (COCOON)}, + series = {Lecture Notes in Computer Science}, + volume = {7434}, + pages = {396--407}, + publisher = {Springer}, + address = {Berlin, Heidelberg}, + year = {2012}, + doi = {10.1007/978-3-642-32241-9_34}, + url = {https://doi.org/10.1007/978-3-642-32241-9_34} +} + +@article{NavarroSadakane2014, + author = {Navarro, Gonzalo and Sadakane, Kunihiko}, + title = {Fully Functional Static and Dynamic Succinct Trees}, + journal = {ACM Transactions on Algorithms}, + volume = {10}, + number = {3}, + pages = {16:1--16:39}, + year = {2014}, + doi = {10.1145/2601073}, + url = {https://doi.org/10.1145/2601073} +} + +@article{DemaineLandauWeimann2014, + author = {Demaine, Erik D. and Landau, Gad M. and Weimann, Oren}, + title = {On Cartesian Trees and Range Minimum Queries}, + journal = {Algorithmica}, + volume = {68}, + number = {3}, + pages = {610--625}, + year = {2014}, + doi = {10.1007/s00453-012-9683-x}, + url = {https://doi.org/10.1007/s00453-012-9683-x} +} + +@article{KowalskiGrabowski2018, + author = {Kowalski, Tomasz M. and Grabowski, Szymon}, + title = {Faster Range Minimum Queries}, + journal = {Software: Practice and Experience}, + volume = {48}, + number = {11}, + pages = {2043--2060}, + year = {2018}, + doi = {10.1002/spe.2597}, + url = {https://doi.org/10.1002/spe.2597} +} + +@inproceedings{AlzamelEtAl2018, + author = {Alzamel, Mai and Charalampopoulos, Panagiotis and Iliopoulos, Costas S. and Pissis, Solon P.}, + title = {How to Answer a Small Batch of {RMQ}s or {LCA} Queries in Practice}, + booktitle = {Combinatorial Algorithms (IWOCA)}, + series = {Lecture Notes in Computer Science}, + volume = {10979}, + pages = {343--355}, + publisher = {Springer}, + address = {Cham, Switzerland}, + year = {2018}, + doi = {10.1007/978-3-319-78825-8_28}, + url = {https://doi.org/10.1007/978-3-319-78825-8_28} +} + +@inproceedings{LiuYu2020, + author = {Liu, Mingmou and Yu, Huacheng}, + title = {Lower Bound for Succinct Range Minimum Query}, + booktitle = {Proceedings of the 52nd Annual ACM SIGACT Symposium on Theory of Computing (STOC)}, + pages = {1402--1415}, + publisher = {ACM}, + address = {New York, NY, USA}, + year = {2020}, + doi = {10.1145/3357713.3384260}, + url = {https://doi.org/10.1145/3357713.3384260} +} + +@article{Russo2022, + author = {Russo, Lu{\'\i}s M. S.}, + title = {Range Minimum Queries in Minimal Space}, + journal = {Theoretical Computer Science}, + volume = {909}, + pages = {19--38}, + year = {2022}, + doi = {10.1016/j.tcs.2022.01.025}, + url = {https://doi.org/10.1016/j.tcs.2022.01.025} +} + +@article{Sadakane2007, + author = {Sadakane, Kunihiko}, + title = {Compressed Suffix Trees with Full Functionality}, + journal = {Theory of Computing Systems}, + volume = {41}, + number = {4}, + pages = {589--607}, + year = {2007}, + doi = {10.1007/s00224-006-1198-x}, + url = {https://doi.org/10.1007/s00224-006-1198-x} +} + +@inproceedings{Muthukrishnan2002, + author = {Muthukrishnan, S.}, + title = {Efficient Algorithms for Document Retrieval Problems}, + booktitle = {Proceedings of the Thirteenth Annual ACM-SIAM Symposium on Discrete Algorithms (SODA)}, + pages = {657--666}, + publisher = {ACM/SIAM}, + address = {New York, NY, USA}, + year = {2002}, + doi = {10.1145/545381.545469}, + url = {https://doi.org/10.1145/545381.545469} +} + @misc{Slotin2022SegmentTrees, author = {Slotin, Sergey}, title = {Segment Trees}, @@ -223,3 +570,345 @@ @article{geary2006 year = {2006}, doi = {10.1016/j.tcs.2008.06.016} } + +% --- Compact Integer Sets and Vectors --- + +@article{Elias1974, + author = {Elias, Peter}, + title = {Efficient Storage and Retrieval by Content and Address of Static Files}, + journal = {Journal of the ACM}, + volume = {21}, + number = {2}, + pages = {246--260}, + year = {1974}, + doi = {10.1145/321812.321820}, + url = {https://doi.org/10.1145/321812.321820} +} + +@techreport{Fano1971, + author = {Fano, Robert Mario}, + title = {On the Number of Bits Required to Implement an Associative Memory}, + institution = {Computer Structures Group, Project MAC, Massachusetts Institute of Technology}, + number = {Memorandum 61}, + address = {Cambridge, MA, USA}, + year = {1971}, + url = {http://csg.csail.mit.edu/pubs/memos/Memo-61/Memo-61.pdf} +} + +@article{Elias1975, + author = {Elias, Peter}, + title = {Universal Codeword Sets and Representations of the Integers}, + journal = {IEEE Transactions on Information Theory}, + volume = {21}, + number = {2}, + pages = {194--203}, + year = {1975}, + doi = {10.1109/TIT.1975.1055349}, + url = {https://doi.org/10.1109/TIT.1975.1055349} +} + +@article{Golomb1966, + author = {Golomb, Solomon W.}, + title = {Run-Length Encodings}, + journal = {IEEE Transactions on Information Theory}, + volume = {12}, + number = {3}, + pages = {399--401}, + year = {1966}, + doi = {10.1109/TIT.1966.1053907}, + url = {https://doi.org/10.1109/TIT.1966.1053907} +} + +@article{RamanEtAl2007, + author = {Raman, Rajeev and Raman, Venkatesh and Satti, Srinivasa Rao}, + title = {Succinct Indexable Dictionaries with Applications to Encoding {$k$}-ary Trees, Prefix Sums and Multisets}, + journal = {ACM Transactions on Algorithms}, + volume = {3}, + number = {4}, + pages = {43:1--43:25}, + year = {2007}, + doi = {10.1145/1290672.1290680}, + url = {https://doi.org/10.1145/1290672.1290680} +} + +@inproceedings{OkanoharaSadakane2007, + author = {Okanohara, Daisuke and Sadakane, Kunihiko}, + title = {Practical Entropy-Compressed Rank/Select Dictionary}, + booktitle = {Proceedings of the Ninth Workshop on Algorithm Engineering and Experiments (ALENEX)}, + pages = {60--70}, + publisher = {SIAM}, + year = {2007}, + doi = {10.1137/1.9781611972870.6}, + url = {https://doi.org/10.1137/1.9781611972870.6} +} + +@inproceedings{Vigna2013, + author = {Vigna, Sebastiano}, + title = {Quasi-Succinct Indices}, + booktitle = {Proceedings of the Sixth ACM International Conference on Web Search and Data Mining (WSDM)}, + pages = {83--92}, + publisher = {ACM}, + year = {2013}, + doi = {10.1145/2433396.2433409}, + url = {https://doi.org/10.1145/2433396.2433409} +} + +@article{MoffatStuiver2000, + author = {Moffat, Alistair and Stuiver, Lang}, + title = {Binary Interpolative Coding for Effective Index Compression}, + journal = {Information Retrieval}, + volume = {3}, + number = {1}, + pages = {25--47}, + year = {2000}, + doi = {10.1023/A:1013002601898}, + url = {https://doi.org/10.1023/A:1013002601898} +} + +@inproceedings{OttavianoVenturini2014, + author = {Ottaviano, Giuseppe and Venturini, Rossano}, + title = {Partitioned Elias--Fano Indexes}, + booktitle = {Proceedings of the 37th International ACM SIGIR Conference on Research and Development in Information Retrieval}, + pages = {273--282}, + publisher = {ACM}, + year = {2014}, + doi = {10.1145/2600428.2609615}, + url = {https://doi.org/10.1145/2600428.2609615} +} + +@article{PibiriVenturini2021, + author = {Pibiri, Giulio Ermanno and Venturini, Rossano}, + title = {Techniques for Inverted Index Compression}, + journal = {ACM Computing Surveys}, + volume = {53}, + number = {6}, + pages = {125:1--125:36}, + year = {2021}, + doi = {10.1145/3415148}, + url = {https://doi.org/10.1145/3415148} +} + +@article{ChambiEtAl2016, + author = {Chambi, Samy and Lemire, Daniel and Kaser, Owen and Godin, Robert}, + title = {Better Bitmap Performance with Roaring Bitmaps}, + journal = {Software: Practice and Experience}, + volume = {46}, + number = {5}, + pages = {709--719}, + year = {2016}, + doi = {10.1002/spe.2325}, + url = {https://doi.org/10.1002/spe.2325} +} + +@article{LemireEtAl2018Roaring, + author = {Lemire, Daniel and Kaser, Owen and Kurz, Nathan and Deri, Luca and O'Hara, Chris and Saint-Jacques, Fran{\c{c}}ois and Ssi-Yan-Kai, Gregory}, + title = {Roaring Bitmaps: Implementation of an Optimized Software Library}, + journal = {Software: Practice and Experience}, + volume = {48}, + number = {4}, + pages = {867--895}, + year = {2018}, + doi = {10.1002/spe.2560}, + url = {https://doi.org/10.1002/spe.2560} +} + +@article{LemireBoytsov2015, + author = {Lemire, Daniel and Boytsov, Leonid}, + title = {Decoding Billions of Integers per Second through Vectorization}, + journal = {Software: Practice and Experience}, + volume = {45}, + number = {1}, + pages = {1--29}, + year = {2015}, + doi = {10.1002/spe.2203}, + url = {https://doi.org/10.1002/spe.2203} +} + +@article{LemireKurzRupp2018, + author = {Lemire, Daniel and Kurz, Nathan and Rupp, Christoph}, + title = {Stream {VByte}: Faster Byte-Oriented Integer Compression}, + journal = {Information Processing Letters}, + volume = {130}, + pages = {1--6}, + year = {2018}, + doi = {10.1016/j.ipl.2017.09.011}, + url = {https://doi.org/10.1016/j.ipl.2017.09.011} +} + +@article{DammeEtAl2019, + author = {Damme, Patrick and Ungeth{\"u}m, Annett and Hildebrandt, Juliana and Habich, Dirk and Lehner, Wolfgang}, + title = {From a Comprehensive Experimental Survey to a Cost-Based Selection Strategy for Lightweight Integer Compression Algorithms}, + journal = {ACM Transactions on Database Systems}, + volume = {44}, + number = {3}, + pages = {18:1--18:46}, + year = {2019}, + doi = {10.1145/3323991}, + url = {https://doi.org/10.1145/3323991} +} + +@article{BrisaboaEtAl2013, + author = {Brisaboa, Nieves R. and Ladra, Susana and Navarro, Gonzalo}, + title = {{DACs}: Bringing Direct Access to Variable-Length Codes}, + journal = {Information Processing \& Management}, + volume = {49}, + number = {1}, + pages = {392--404}, + year = {2013}, + doi = {10.1016/j.ipm.2012.08.003}, + url = {https://doi.org/10.1016/j.ipm.2012.08.003} +} + +@article{ClaudeEtAl2015, + author = {Claude, Francisco and Navarro, Gonzalo and Ord{\'o}{\~n}ez, Alberto}, + title = {The Wavelet Matrix: An Efficient Wavelet Tree for Large Alphabets}, + journal = {Information Systems}, + volume = {47}, + pages = {15--32}, + year = {2015}, + doi = {10.1016/j.is.2014.06.002}, + url = {https://doi.org/10.1016/j.is.2014.06.002} +} + +@article{FerraginaEtAl2007, + author = {Ferragina, Paolo and Manzini, Giovanni and M{\"a}kinen, Veli and Navarro, Gonzalo}, + title = {Compressed Representations of Sequences and Full-Text Indexes}, + journal = {ACM Transactions on Algorithms}, + volume = {3}, + number = {2}, + pages = {20:1--20:24}, + year = {2007}, + doi = {10.1145/1240233.1240243}, + url = {https://doi.org/10.1145/1240233.1240243} +} + +@article{FerraginaVenturini2007, + author = {Ferragina, Paolo and Venturini, Rossano}, + title = {A Simple Storage Scheme for Strings Achieving Entropy Bounds}, + journal = {Theoretical Computer Science}, + volume = {372}, + number = {1}, + pages = {115--121}, + year = {2007}, + doi = {10.1016/j.tcs.2006.12.012}, + url = {https://doi.org/10.1016/j.tcs.2006.12.012} +} + +@inproceedings{PibiriVenturini2017, + author = {Pibiri, Giulio Ermanno and Venturini, Rossano}, + title = {Dynamic Elias--Fano Representation}, + booktitle = {28th Annual Symposium on Combinatorial Pattern Matching (CPM)}, + series = {Leibniz International Proceedings in Informatics (LIPIcs)}, + volume = {78}, + pages = {30:1--30:14}, + publisher = {Schloss Dagstuhl -- Leibniz-Zentrum f{\"u}r Informatik}, + year = {2017}, + doi = {10.4230/LIPIcs.CPM.2017.30}, + url = {https://doi.org/10.4230/LIPIcs.CPM.2017.30} +} + +@inproceedings{PatrascuThorup2014, + author = {P{\u{a}}tra{\c{s}}cu, Mihai and Thorup, Mikkel}, + title = {Dynamic Integer Sets with Optimal Rank, Select, and Predecessor Search}, + booktitle = {55th IEEE Annual Symposium on Foundations of Computer Science (FOCS)}, + pages = {166--175}, + publisher = {IEEE}, + year = {2014}, + doi = {10.1109/FOCS.2014.26}, + url = {https://doi.org/10.1109/FOCS.2014.26} +} + +@inproceedings{BlandfordBlelloch2004, + author = {Blandford, Daniel K. and Blelloch, Guy E.}, + title = {Compact Representations of Ordered Sets}, + booktitle = {Proceedings of the Fifteenth Annual ACM-SIAM Symposium on Discrete Algorithms (SODA)}, + pages = {11--19}, + publisher = {SIAM}, + year = {2004}, + url = {https://www.cs.cmu.edu/~guyb/papers/BB04.pdf} +} + +@article{NavarroNekrich2014, + author = {Navarro, Gonzalo and Nekrich, Yakov}, + title = {Optimal Dynamic Sequence Representations}, + journal = {SIAM Journal on Computing}, + volume = {43}, + number = {5}, + pages = {1781--1806}, + year = {2014}, + doi = {10.1137/130908245}, + url = {https://doi.org/10.1137/130908245} +} + +@inproceedings{MunroNekrich2015, + author = {Munro, J. Ian and Nekrich, Yakov}, + title = {Compressed Data Structures for Dynamic Sequences}, + booktitle = {23rd Annual European Symposium on Algorithms (ESA)}, + series = {Lecture Notes in Computer Science}, + volume = {9294}, + pages = {891--902}, + publisher = {Springer}, + year = {2015}, + doi = {10.1007/978-3-662-48350-3_74}, + url = {https://doi.org/10.1007/978-3-662-48350-3_74} +} + +@inproceedings{JanssonEtAl2012, + author = {Jansson, Jesper and Sadakane, Kunihiko and Sung, Wing-Kin}, + title = {{CRAM}: Compressed Random Access Memory}, + booktitle = {Automata, Languages, and Programming (ICALP)}, + series = {Lecture Notes in Computer Science}, + volume = {7391}, + pages = {510--521}, + publisher = {Springer}, + year = {2012}, + doi = {10.1007/978-3-642-31594-7_43}, + url = {https://doi.org/10.1007/978-3-642-31594-7_43} +} + +@inproceedings{Prezza2017, + author = {Prezza, Nicola}, + title = {A Framework of Dynamic Data Structures for String Processing}, + booktitle = {16th International Symposium on Experimental Algorithms (SEA)}, + series = {Leibniz International Proceedings in Informatics (LIPIcs)}, + volume = {75}, + pages = {11:1--11:15}, + publisher = {Schloss Dagstuhl -- Leibniz-Zentrum f{\"u}r Informatik}, + year = {2017}, + doi = {10.4230/LIPIcs.SEA.2017.11}, + url = {https://doi.org/10.4230/LIPIcs.SEA.2017.11} +} + +@inproceedings{DongesEtAl2022, + author = {D{\"o}nges, Saska and Puglisi, Simon J. and Raman, Rajeev}, + title = {On Dynamic Bitvector Implementations}, + booktitle = {2022 Data Compression Conference (DCC)}, + pages = {252--261}, + publisher = {IEEE}, + year = {2022}, + doi = {10.1109/DCC52660.2022.00033}, + url = {https://doi.org/10.1109/DCC52660.2022.00033} +} + +@article{Navarro2025, + author = {Navarro, Gonzalo}, + title = {(Worst-Case) Optimal Adaptive Dynamic Bitvectors}, + journal = {Theory of Computing Systems}, + volume = {69}, + number = {3}, + pages = {30}, + year = {2025}, + doi = {10.1007/s00224-025-10229-8}, + url = {https://doi.org/10.1007/s00224-025-10229-8} +} + +@inproceedings{KuszmaulEtAl2026, + author = {Kuszmaul, William and Liang, Jingxun and Zhou, Renfei}, + title = {Succinct Dynamic Rank/Select: Bypassing the Tree-Structure Bottleneck}, + booktitle = {Proceedings of the 2026 Annual ACM-SIAM Symposium on Discrete Algorithms (SODA)}, + pages = {3760--3804}, + publisher = {SIAM}, + year = {2026}, + doi = {10.1137/1.9781611978971.138}, + url = {https://doi.org/10.1137/1.9781611978971.138} +} diff --git a/academic/notes/compact-integer-sets-vectors/.gitignore b/academic/notes/compact-integer-sets-vectors/.gitignore new file mode 100644 index 0000000..ad29309 --- /dev/null +++ b/academic/notes/compact-integer-sets-vectors/.gitignore @@ -0,0 +1,2 @@ +/.quarto/ +**/*.quarto_ipynb diff --git a/agentic/cpp b/agentic/cpp index c723c1e..2f48ca4 160000 --- a/agentic/cpp +++ b/agentic/cpp @@ -1 +1 @@ -Subproject commit c723c1e81354f3d40ad00766f432bf66e5c27a2b +Subproject commit 2f48ca47f10dae2de7498337d9a265ebde65f413 diff --git a/include/pixie/bits.h b/include/pixie/bits.h index ed03503..5108137 100644 --- a/include/pixie/bits.h +++ b/include/pixie/bits.h @@ -17,7 +17,7 @@ #define PIXIE_SSE41_SUPPORT #endif -#if defined(PIXIE_AVX512_SUPPORT) || defined(PIXIE_BMI2_SUPPORT) || \ +#if defined(__AVX512F__) || defined(PIXIE_BMI2_SUPPORT) || \ defined(PIXIE_AVX2_SUPPORT) || defined(PIXIE_SSE41_SUPPORT) #include #endif @@ -279,6 +279,104 @@ static inline uint64_t first_bits_mask(size_t num) { return num >= 64 ? UINT64_MAX : ((1llu << num) - 1); } +namespace pixie { + +/** + * @brief Copy LSB-first packed bits between disjoint word arrays. + * + * @details Copies [source_bit, source_bit + count) to + * [destination_bit, destination_bit + count), preserving all other destination + * bits. Offsets are zero-based. The caller must supply valid, nonoverlapping + * backing word arrays covering both ranges; invalid input is undefined + * behavior. Only words intersecting the ranges are accessed, with no SIMD + * alignment or extra padding requirement. A zero count does not access either + * pointer. Does not allocate or retain pointers. In-place callers must stage + * through separate scratch storage before writing back. + * @param source Source word array; borrowed only for this call. + * @param source_bit Zero-based inclusive start of the source bit range. + * @param destination Destination word array; borrowed only for this call. + * @param destination_bit Zero-based inclusive start of the destination bit + * range. + * @param count Number of bits to copy; zero permits null pointers. + */ +inline void copy_packed_bits(const uint64_t* source, + size_t source_bit, + uint64_t* destination, + size_t destination_bit, + size_t count) noexcept { + // Peel the destination's partial word, then write whole words in batches. + const auto copy_boundary = [&](size_t width) { + const auto shift = source_bit % 64; + uint64_t value = source[source_bit / 64] >> shift; + if (width > 64 - shift) { + value |= source[source_bit / 64 + 1] << (64 - shift); + } + const auto offset = destination_bit % 64; + const auto mask = first_bits_mask(width) << offset; + auto& word = destination[destination_bit / 64]; + word = (word & ~mask) | ((value << offset) & mask); + source_bit += width; + destination_bit += width; + count -= width; + }; + if (count == 0) { + return; + } + if (destination_bit % 64 != 0) { + copy_boundary(std::min(count, 64 - destination_bit % 64)); + } + const auto shift = source_bit % 64; + const auto* input = source + source_bit / 64; + auto* output = destination + destination_bit / 64; + auto words = count / 64; + if (shift == 0) { + std::copy_n(input, words, output); + } else { +#if defined(__AVX512F__) +#if defined(__AVX512VBMI2__) + const auto shifts = _mm512_set1_epi64(shift); +#else + const auto low_shift = _mm_cvtsi64_si128(shift); + const auto high_shift = _mm_cvtsi64_si128(64 - shift); +#endif + for (; words >= 8; words -= 8, input += 8, output += 8) { + const auto low = _mm512_loadu_si512(input); + const auto high = _mm512_loadu_si512(input + 1); +#if defined(__AVX512VBMI2__) + _mm512_storeu_si512(output, _mm512_shrdv_epi64(low, high, shifts)); +#else + _mm512_storeu_si512(output, + _mm512_or_si512(_mm512_srl_epi64(low, low_shift), + _mm512_sll_epi64(high, high_shift))); +#endif + } +#elif defined(PIXIE_AVX2_SUPPORT) + const auto low_shift = _mm_cvtsi64_si128(shift); + const auto high_shift = _mm_cvtsi64_si128(64 - shift); + for (; words >= 4; words -= 4, input += 4, output += 4) { + const auto low = + _mm256_loadu_si256(reinterpret_cast(input)); + const auto high = + _mm256_loadu_si256(reinterpret_cast(input + 1)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(output), + _mm256_or_si256(_mm256_srl_epi64(low, low_shift), + _mm256_sll_epi64(high, high_shift))); + } +#endif + for (size_t i = 0; i < words; ++i) { + output[i] = (input[i] >> shift) | (input[i + 1] << (64 - shift)); + } + } + source_bit += count / 64 * 64; + destination_bit += count / 64 * 64; + count %= 64; + if (count != 0) { + copy_boundary(count); + } +} + +} // namespace pixie + /** * @brief Number of 1 bits in positions 0 .. count - 1 * @details Assumes count @@ -300,7 +398,7 @@ static inline uint64_t first_bits_mask(size_t num) { * 64 bits and then reduce_add to sum the result. */ static inline uint64_t rank_512(const uint64_t* x, uint64_t count) { -#ifdef PIXIE_AVX512_SUPPORT +#if defined(PIXIE_AVX512_SUPPORT) && defined(__AVX512VBMI2__) __m512i a = _mm512_maskz_set1_epi64((1ull << ((count >> 6))) - 1, std::numeric_limits::max()); diff --git a/include/pixie/detail/sequence/bit_block.h b/include/pixie/detail/sequence/bit_block.h new file mode 100644 index 0000000..0791348 --- /dev/null +++ b/include/pixie/detail/sequence/bit_block.h @@ -0,0 +1,205 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/// @cond PIXIE_SEQUENCE_INTERNAL +namespace pixie::detail::sequence { + +/** + * @brief Standalone, fixed-capacity, circular packed-bit block. + * @details Owns typed words without allocation. The ring wraps at size(), never + * at capacity. Metadata and CacheLine alignment padding are additional to the + * payload budget. Copies own independent contents. Local mutations require + * valid arguments and never allocate or throw; checked entry points are + * supplied by SequenceTree. Unused bits are not logical elements. + * @tparam PayloadBits Payload capacity, a positive multiple of 64. + */ +template +class alignas(CacheLine) BitBlock { + static_assert(PayloadBits >= 64 && PayloadBits % 64 == 0); + static_assert(PayloadBits <= std::numeric_limits::max() / 2); + + public: + /** @brief Indexed reads return bits by value. */ + using value_type = bool; + /** @brief Maximum valid element count. */ + static constexpr std::size_t capacity = PayloadBits; + /** @brief Typed, LSB-first payload words. */ + using Payload = std::array; + /** @brief Byte offset of the payload; allocation alignment is alignof(Block). + */ + static constexpr std::size_t payload_offset_bytes = 2 * sizeof(std::uint64_t); + /** @brief Payload requires word alignment, not separate CacheLine alignment. + */ + static constexpr std::size_t payload_alignment = alignof(std::uint64_t); + + /** @brief Construct an empty block without allocation. */ + BitBlock() noexcept = default; + /** + * @brief Copy n LSB-first bits, ignoring extra words and dirty high padding. + * @details The source need not outlive this block; initial origin is zero. + * @param words Source word span. + * @param n Logical count in [0, capacity]. + * @throws std::invalid_argument If n exceeds capacity or words is too short. + */ + BitBlock(std::span words, std::size_t n) { + if (n > capacity || n / 64 + (n % 64 != 0) > words.size()) { + throw std::invalid_argument("BitBlock: input size"); + } + n_ = n; + std::copy_n(words.begin(), n / 64 + (n % 64 != 0), bits_.begin()); + } + /** @brief Return the valid bit count. @return Count excluding padding. */ + std::size_t size() const noexcept { return n_; } + /** @brief Test emptiness. @return Whether size() is zero. */ + bool empty() const noexcept { return n_ == 0; } + /** + * @brief Read a bit by value. + * @param i Zero-based index in [0, size()). + * @return Logical bit, not a writable reference. + * @throws std::out_of_range If i >= size(). + */ + bool operator[](std::size_t i) const { + if (i >= n_) { + throw std::out_of_range("BitBlock: index"); + } + const auto p = (h_ + i) % n_; + return (bits_[p / 64] >> (p % 64)) & 1; + } + /** + * @brief Read at most 64 consecutive logical bits without flattening. + * @details Requires offset <= size() and width <= min(64, size()-offset). + * A zero width returns zero, including on an empty block. The result is + * LSB-first with zero high padding. Reads only words containing logical bits, + * even across the circular boundary; width 64 never shifts by 64. Invalid + * arguments violate the primitive precondition (asserted in debug builds). + * @param offset First logical bit. + * @param width Number of bits to decode. + * @return Unsigned field value. + */ + std::uint64_t read_bits(std::size_t offset, + std::size_t width) const noexcept { + assert(offset <= n_ && width <= 64 && width <= n_ - offset); + if (width == 0) { + return 0; + } + const auto physical = (h_ + offset) % n_; + const auto first = std::min(width, n_ - physical); + auto read = [&](std::size_t p, std::size_t count) { + const auto shift = p % 64; + auto value = bits_[p / 64] >> shift; + if (count > 64 - shift) { + value |= bits_[p / 64 + 1] << (64 - shift); + } + return count == 64 ? value : value & ((std::uint64_t{1} << count) - 1); + }; + const auto value = read(physical, first); + return first == width ? value : value | (read(0, width - first) << first); + } + /** + * @brief Materialize logical words without changing representation. + * @return LSB-first words with all padding zero. + */ + Payload flatten() const noexcept { + Payload out{}; + copy(0, out, 0, n_); + return out; + } + /** + * @brief Rotate a valid half-open local interval without allocation. + * @details Requires left <= right <= size(). Empty ranges are no-ops; a whole + * block rotation adjusts only the origin. Invalid arguments violate the + * primitive precondition (asserted in debug builds). + * @param left Inclusive start. + * @param right Exclusive end. + * @param distance Left distance, reduced modulo the nonempty range length. + */ + void rotate_left(std::size_t left, + std::size_t right, + std::size_t distance) noexcept { + assert(left <= right && right <= n_); + const auto length = right - left; + if (length == 0 || (distance %= length) == 0) { + return; + } + if (length == n_) { + h_ = (h_ + distance) % n_; + return; + } + Payload scratch{}; + copy(left + distance, scratch, 0, length - distance); + copy(left, scratch, length - distance, distance); + const auto p = (h_ + left) % n_; + const auto prefix = std::min(length, n_ - p); + copy_packed_bits(scratch.data(), 0, bits_.data(), p, prefix); + copy_packed_bits(scratch.data(), prefix, bits_.data(), 0, length - prefix); + } + /** + * @brief Repartition two blocks, preserving their concatenated logical bits. + * @details Requires distinct blocks, left_size <= capacity, and + * left_size <= size()+rhs.size() <= left_size+capacity. Nonallocating and + * nonthrowing. Both circular origins may normalize. An empty rhs permits + * splitting; left_size equal to the total permits combining. + * @param rhs Right block; receives the remaining suffix. + * @param left_size Desired valid count in this block. + */ + void redistribute(BitBlock& rhs, std::size_t left_size) noexcept { + assert(this != &rhs); + const auto total = n_ + rhs.n_; + assert(left_size <= capacity && left_size <= total && + total - left_size <= capacity); + Payload a{}, b{}; + auto gather = [&](std::size_t start, Payload& out, std::size_t count) { + const auto first = start < n_ ? std::min(count, n_ - start) : 0; + copy(start, out, 0, first); + rhs.copy(start + first - std::min(start + first, n_), out, first, + count - first); + }; + gather(0, a, left_size); + gather(left_size, b, total - left_size); + bits_ = a; + rhs.bits_ = b; + n_ = left_size; + rhs.n_ = total - left_size; + h_ = rhs.h_ = 0; + } + /** @brief Report fixed payload storage. @return Bytes including unused bits. + */ + std::size_t payload_capacity_bytes() const noexcept { return sizeof(bits_); } + /** @brief Report nonpayload storage. @return Metadata and alignment padding. + */ + std::size_t metadata_bytes() const noexcept { + return sizeof(*this) - sizeof(bits_); + } + + private: + void copy(std::size_t offset, + Payload& out, + std::size_t destination, + std::size_t count) const noexcept { + if (count == 0) { + return; + } + const auto p = (h_ + offset) % n_; + const auto prefix = std::min(count, n_ - p); + copy_packed_bits(bits_.data(), p, out.data(), destination, prefix); + copy_packed_bits(bits_.data(), 0, out.data(), destination + prefix, + count - prefix); + } + std::uint64_t n_ = 0; + std::uint64_t h_ = 0; + Payload bits_{}; +}; + +} // namespace pixie::detail::sequence +/// @endcond diff --git a/include/pixie/detail/sequence/packed_bit_block.h b/include/pixie/detail/sequence/packed_bit_block.h new file mode 100644 index 0000000..9719709 --- /dev/null +++ b/include/pixie/detail/sequence/packed_bit_block.h @@ -0,0 +1,27 @@ +#pragma once + +#include + +/// @cond PIXIE_SEQUENCE_INTERNAL +namespace pixie::detail::sequence { + +/** + * @brief Bit block whose complete aligned object fits a physical-bit budget. + * @details Two uint64_t fields hold valid length and circular origin; all + * remaining whole words are payload. There are no hidden tree fields or dynamic + * allocations. On an eight-bit-byte platform PackedBitBlock<2048> is exactly + * 256 bytes and holds 1920 bits. Local contracts are inherited from BitBlock. + * @tparam StorageBits Complete object budget, a positive CacheLine multiple. + */ +template +class PackedBitBlock : public BitBlock { + static_assert(StorageBits >= kAlignedStorageLineBits && + StorageBits % kAlignedStorageLineBits == 0); + static_assert(sizeof(BitBlock) == StorageBits / 8); + + public: + using BitBlock::BitBlock; +}; + +} // namespace pixie::detail::sequence +/// @endcond diff --git a/include/pixie/detail/sequence/packed_value_block.h b/include/pixie/detail/sequence/packed_value_block.h new file mode 100644 index 0000000..e8f2940 --- /dev/null +++ b/include/pixie/detail/sequence/packed_value_block.h @@ -0,0 +1,178 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +/// @cond PIXIE_SEQUENCE_INTERNAL +namespace pixie::detail::sequence { + +/** + * @brief Fixed-width unsigned values over an owning circular packed bit block. + * @details No adapter metadata or allocation: the underlying block records the + * bit count and origin. Logical boundaries are multiples of Width. Copies own + * independent data; reads return values. Local mutations require valid inputs + * and never allocate or throw. Narrow Width is an unsupported internal + * control, not a codec: construction rejects values which do not fit. + * @tparam T Bool or an unsigned integer with at most 64 value bits. + * @tparam StorageBits Complete local block budget, not a tree leaf budget. + * @tparam Width Positive field width, at most the value width of T. + */ +template ::digits> +class PackedValueBlock { + static_assert(std::is_integral_v && std::is_unsigned_v && + std::numeric_limits::digits <= 64); + static_assert(Width > 0 && Width <= std::numeric_limits::digits); + using Bits = PackedBitBlock; + + public: + /** @brief Immutable indexed result. @details Includes bool by value. */ + using value_type = T; + /** @brief Maximum element count. @details Unused trailing bits are padding. + */ + static constexpr std::size_t capacity = Bits::capacity / Width; + /** @brief Stored bits per value. @details Independent of positional counts. + */ + static constexpr std::size_t width = Width; + /** @brief Payload byte offset. @details Inherited from the sole member. */ + static constexpr std::size_t payload_offset_bytes = + Bits::payload_offset_bytes; + /** @brief Payload alignment. @details The block itself is cache-line aligned. + */ + static constexpr std::size_t payload_alignment = Bits::payload_alignment; + static_assert(capacity > 0); + + /** @brief Construct empty. @details Performs no allocation. */ + PackedValueBlock() noexcept = default; + /** + * @brief Copy values into independent bounded storage. + * @details The source is needed only during construction; initial origin is + * zero. + * @param values At most capacity representable fields. + * @throws std::invalid_argument If count or a value exceeds its capacity. + */ + explicit PackedValueBlock(std::span values) { + if (values.size() > capacity) { + throw std::invalid_argument("PackedValueBlock: input size"); + } + typename Bits::Payload words{}; + for (std::size_t i = 0; i < values.size(); ++i) { + if (static_cast(values[i]) > field_max) { + throw std::invalid_argument("PackedValueBlock: value width"); + } + encode(words, i, values[i]); + } + bits_ = Bits(words, values.size() * Width); + } + /** @brief Return element count. @details Excludes unused trailing bits. + * @return Number of logical values. + */ + std::size_t size() const noexcept { return bits_.size() / Width; } + /** @brief Test emptiness. @details Equivalent to size() == 0. + * @return Whether the block contains no values. + */ + bool empty() const noexcept { return bits_.empty(); } + /** + * @brief Read a logical value without flattening the payload. + * @details Decodes at most 64 bits, including wrapped fields. + * @param i Zero-based element position. + * @return The decoded value at i. + * @throws std::out_of_range If i >= size(). + */ + T operator[](std::size_t i) const { + if (i >= size()) { + throw std::out_of_range("PackedValueBlock: index"); + } + if constexpr (Width == 1) { + return static_cast(bits_[i]); + } else { + return static_cast(bits_.read_bits(i * Width, Width)); + } + } + /** + * @brief Rotate a valid half-open element interval without allocation. + * @details Requires left <= right <= size(). Reduces distance in element + * units before converting to bits; empty intervals do nothing. + * @param left Inclusive starting element position. + * @param right Exclusive ending element position. + * @param distance Leftward distance in elements, reduced modulo length. + */ + void rotate_left(std::size_t left, + std::size_t right, + std::size_t distance) noexcept { + assert(left <= right && right <= size()); + if constexpr (Width == 1) { + bits_.rotate_left(left, right, distance); + } else if (left != right) { + bits_.rotate_left(left * Width, right * Width, + (distance % (right - left)) * Width); + } + } + /** + * @brief Repartition concatenated values between distinct blocks. + * @details Requires left_size <= capacity, left_size <= size()+rhs.size(), + * and size()+rhs.size()-left_size <= capacity. Preserves order, normalizes + * origins, and neither allocates nor throws. + * @param rhs Distinct following block; receives the remaining suffix. + * @param left_size Desired number of elements in this block afterward. + */ + void redistribute(PackedValueBlock& rhs, std::size_t left_size) noexcept { + assert(this != &rhs && left_size <= capacity && + left_size <= size() + rhs.size() && + size() + rhs.size() - left_size <= capacity); + bits_.redistribute(rhs.bits_, left_size * Width); + } + /** + * @brief Materialize a nonnegative index bias using bounded stack scratch. + * @details Requires every decoded value plus bias to fit Width. Normalizes + * the origin; does not allocate or throw. This primitive supports tagged + * permutation leaves, not a public arbitrary transformation on a container. + * @param bias Nonnegative value added to each logical field. + */ + void add_bias(std::uint64_t bias) noexcept { + typename Bits::Payload words{}; + const auto n = size(); + for (std::size_t i = 0; i < n; ++i) { + const auto value = bits_.read_bits(i * Width, Width); + assert(bias <= field_max && value <= field_max - bias); + encode(words, i, value + bias); + } + bits_ = Bits(words, n * Width); + } + /** @brief Report payload capacity. @details Includes field/word slack. + * @return Bytes reserved for packed payload. + */ + std::size_t payload_capacity_bytes() const noexcept { + return bits_.payload_capacity_bytes(); + } + /** @brief Report metadata bytes. @details Includes local alignment padding. + * @return Non-payload bytes in the complete block object. + */ + std::size_t metadata_bytes() const noexcept { return bits_.metadata_bytes(); } + + private: + static constexpr std::uint64_t field_max = + std::numeric_limits::max() >> (64 - Width); + static void encode(typename Bits::Payload& words, + std::size_t i, + std::uint64_t value) noexcept { + const auto bit = i * Width; + const auto shift = bit % 64; + words[bit / 64] |= value << shift; + if (Width > 64 - shift) { + words[bit / 64 + 1] |= value >> (64 - shift); + } + } + Bits bits_; +}; + +} // namespace pixie::detail::sequence +/// @endcond diff --git a/include/pixie/detail/sequence/pointer_block.h b/include/pixie/detail/sequence/pointer_block.h new file mode 100644 index 0000000..9b64e71 --- /dev/null +++ b/include/pixie/detail/sequence/pointer_block.h @@ -0,0 +1,165 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/// @cond PIXIE_SEQUENCE_INTERNAL +namespace pixie::detail::sequence { + +/** + * @brief Bounded, nonowning block of native pointers for a SequenceTree. + * @details StorageBits budgets the complete inline object, including length, + * circular origin and alignment padding. Pointers are never encoded as integer + * fields. Local mutations move pointers only, never pointed-to objects, and use + * bounded stack scratch without allocation. The caller owns all pointees. + * @tparam T Pointee type; reads expose const T* without dereferencing it. + * @tparam StorageBits Complete inline budget in bits, a positive + * CacheLine-sized multiple large enough for both metadata fields and at least + * one pointer. + */ +template +class PointerBlock { + static_assert(StorageBits % (sizeof(CacheLine) * CHAR_BIT) == 0); + static_assert(StorageBits / CHAR_BIT >= + 2 * sizeof(std::size_t) + sizeof(const T*)); + + public: + /** + * @brief Immutable-pointee native pointer returned by value. + * @details Does not own or extend the lifetime of the pointed-to object. + */ + using value_type = const T*; + /** + * @brief Actual slot count after paying for both local metadata fields. + * @details Includes only native pointer slots; the complete aligned storage + * object is statically checked against StorageBits. + */ + static constexpr std::size_t capacity = + (StorageBits / CHAR_BIT - 2 * sizeof(std::size_t)) / sizeof(value_type); + static_assert(capacity <= std::numeric_limits::max() / 2); + + private: + struct alignas(CacheLine) Storage { + std::size_t length = 0; + std::size_t origin = 0; + std::array values{}; + }; + static_assert(sizeof(Storage) == StorageBits / CHAR_BIT); + Storage storage_; + + value_type& slot(std::size_t i) noexcept { + return storage_.values[(storage_.origin + i) % storage_.length]; + } + + public: + /** + * @brief Construct an empty block. + * @details Initializes length and origin to zero without allocating. + */ + PointerBlock() noexcept = default; + /** + * @brief Copy pointer values from input, not the pointees. + * @details No input ownership is acquired, and no allocation is performed. + * Copies pointer values in their input order with a zero circular origin. + * @param input Pointer values to copy; the span need not outlive + * construction. + * @throws std::length_error If input.size() > capacity. + */ + explicit PointerBlock(std::span input) { + if (input.size() > capacity) { + throw std::length_error("PointerBlock: capacity"); + } + std::copy(input.begin(), input.end(), storage_.values.begin()); + storage_.length = input.size(); + } + /** + * @brief Return the number of logical pointers. + * @details Constant time and nonthrowing; excludes unused slots. + * @return Pointer count in [0,capacity]. + */ + std::size_t size() const noexcept { return storage_.length; } + /** + * @brief Read a pointer by value at a zero-based position. + * @details Never dereferences the pointer. This unchecked local operation + * asserts its precondition in debug builds; it does not throw. + * @param i Zero-based logical position in [0,size()). + * @return Native pointer value at position i, without acquiring ownership. + * @pre i < size(). + */ + value_type operator[](std::size_t i) const noexcept { + assert(i < size()); + return storage_.values[(storage_.origin + i) % size()]; + } + /** + * @brief Rotate the valid half-open range [left,right) without allocation. + * @details Distance is reduced modulo a nonempty range's length. Empty valid + * ranges are no-ops. Whole-block rotations update only the origin. Other + * elements retain their order, and pointees are never moved or dereferenced. + * @param left Inclusive start of the half-open range. + * @param right Exclusive end of the half-open range. + * @param distance Left-rotation distance in pointer elements. + * @pre left <= right && right <= size(); debug builds assert this condition. + */ + void rotate_left(std::size_t left, + std::size_t right, + std::size_t distance) noexcept { + assert(left <= right && right <= size()); + const auto n = right - left; + if (n == 0 || (distance %= n) == 0) { + return; + } + if (left == 0 && right == size()) { + storage_.origin = (storage_.origin + distance) % size(); + return; + } + auto reverse = [&](std::size_t begin, std::size_t end) { + while (begin < end && begin < --end) { + std::swap(slot(begin++), slot(end)); + } + }; + reverse(left, left + distance); + reverse(left + distance, right); + reverse(left, right); + } + /** + * @brief Redistribute concatenated pointers into two distinct blocks. + * @details Preserves concatenated order, leaving left_count pointers in this + * block and the remaining pointers in right. Resets both circular origins; + * uses at most 2*capacity scratch pointers and cannot allocate or throw. + * @param right Distinct block holding the suffix of the input concatenation. + * @param left_count Desired pointer count in this block after redistribution. + * @pre This block and right are distinct; left_count <= capacity; + * left_count <= size()+right.size(); and size()+right.size()-left_count <= + * capacity. Debug builds assert these conditions. + */ + void redistribute(PointerBlock& right, std::size_t left_count) noexcept { + const auto total = size() + right.size(); + assert(this != &right && left_count <= capacity && left_count <= total && + total - left_count <= capacity); + std::array scratch; + for (std::size_t i = 0; i < size(); ++i) { + scratch[i] = (*this)[i]; + } + for (std::size_t i = 0; i < right.size(); ++i) { + scratch[size() + i] = right[i]; + } + std::copy_n(scratch.begin(), left_count, storage_.values.begin()); + std::copy_n(scratch.begin() + left_count, total - left_count, + right.storage_.values.begin()); + storage_.length = left_count; + right.storage_.length = total - left_count; + storage_.origin = right.storage_.origin = 0; + } +}; + +} // namespace pixie::detail::sequence +/// @endcond diff --git a/include/pixie/detail/sequence/sequence_tree.h b/include/pixie/detail/sequence/sequence_tree.h new file mode 100644 index 0000000..a83a90f --- /dev/null +++ b/include/pixie/detail/sequence/sequence_tree.h @@ -0,0 +1,1338 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef PIXIE_SEQUENCE_TREE_TESTING +#include +#include +#endif + +/// @cond PIXIE_SEQUENCE_INTERNAL +namespace pixie { + +template +class Permutation; +} // namespace pixie + +// Unsupported implementation shared by the two public sequence families. +namespace pixie::detail::sequence { + +/** + * @brief Nonallocating local contract for SequenceTree leaves. + * @details Blocks default-construct empty and own their elements. Capacity is + * positive and at most SIZE_MAX/2. Reads return value_type by value. + * Moves, destruction, size(), and valid local mutations cannot throw. Rotation + * accepts [left,right) within size() and reduces distance modulo its length. + * redistribute requires distinct blocks and a feasible left count; it preserves + * concatenated order, leaves that count on the left, and the rest on the right. + * Neither mutation allocates. These semantic requirements supplement the + * mechanically checked signatures. Throwing element mutations are unsupported. + */ +template +concept SequenceBlock = + std::is_nothrow_default_constructible_v && + std::is_nothrow_move_constructible_v && + std::is_nothrow_move_assignable_v && std::is_nothrow_destructible_v && + requires(B& a, B& b, const B& c, std::size_t n) { + typename B::value_type; + requires B::capacity >= 1 && + B::capacity <= + std::numeric_limits::max() / 2; + { c.size() } noexcept -> std::same_as; + { c[n] } -> std::same_as; + { a.rotate_left(n, n, n) } noexcept -> std::same_as; + { a.redistribute(b, n) } noexcept -> std::same_as; + }; + +/** + * @brief Exclusively owned positional B+ split/join tree over bounded blocks. + * @details All leaves have equal depth. Nonroot nodes have F/2..F children; + * roots have at least two and collapse otherwise. Only the two exterior leaves + * may be below half capacity. Counts are unsigned size_t, up to SIZE_MAX. + * Access is O(F*h); split, join and rotation touch O(F*h) structural entries + * and a constant number of bounded leaf payloads. No normal operation + * enumerates leaves. Allocation preflight is O(h), bounded by the temporary + * node deficit with dismantled nodes recycled; commit is nonallocating under + * SequenceBlock's contract. No sharing, stable references, or concurrent + * mutation is supported. + * + * Internal allocations contain F typed pointer slots and F-1 contiguous + * measures plus a count, aligned to CacheLine. A level-aware move-only owner + * controls roots and detached subtrees; occupied node slots own their children. + * Leaves are separate aligned allocations containing a Block. Optional index + * bias headers exist only when IndexBias is true. A decoded value equals its + * stored field plus the nonnegative biases on its root-to-leaf path. Reads + * accumulate without mutation; dismantling nodes pushes their bias to children, + * and only redistributed leaves materialize fields. The permutation facade + * checks the combined domain before attaching a bias, so every partial sum is + * bounded by the final representable value. No merge-history chain is stored. + * @tparam Block Nonthrowing bounded local block implementation. + * @tparam Fanout Even fanout, at least four (4/8/16 are baseline candidates). + * @tparam Layout Cumulative ends or individual lengths; last length is derived. + * @tparam IndexBias Enable narrowly scoped lazy unsigned index rebasing. Block + * must represent its value_type's full unsigned range and provide nonallocating + * noexcept add_bias(uint64_t), requiring representable resulting values. + */ +template +class SequenceTree { + static_assert(Fanout >= 4 && Fanout % 2 == 0); + static_assert(!IndexBias || + (std::is_unsigned_v && + !std::is_same_v)); + static_assert(!IndexBias || requires(Block& block, std::uint64_t bias) { + { block.add_bias(bias) } noexcept -> std::same_as; + }); + struct NoBias {}; + struct PendingBias { + std::uint64_t value = 0; + }; + using Bias = std::conditional_t; + struct alignas(std::max(alignof(CacheLine), alignof(Block))) Leaf { + Block block; + [[no_unique_address]] Bias bias; + }; + struct alignas(CacheLine) Node { + std::array children{}; + std::array measures{}; + std::size_t count = 0; + [[no_unique_address]] Bias bias; + }; + template + friend class ::pixie::Permutation; + static constexpr std::size_t max_height = + std::numeric_limits::digits; + static constexpr std::size_t minimum_leaf_size = + (static_cast(Block::capacity) + 1) / 2; + + public: + /** @brief Block type accepted by the consuming factory. */ + using block_type = Block; + /** @brief Immutable indexed result type. */ + using value_type = typename Block::value_type; + /** @brief Maximum leaf element count. */ + static constexpr std::size_t block_capacity = Block::capacity; + /** @brief Actual aligned leaf allocation bytes, including block padding. */ + static constexpr std::size_t block_storage_bytes = sizeof(Leaf); + /** @brief Actual aligned internal allocation bytes. */ + static constexpr std::size_t node_storage_bytes = sizeof(Node); + /** @brief Required leaf allocation alignment (payload may differ). */ + static constexpr std::size_t block_alignment = alignof(Leaf); + /** @brief Required internal allocation alignment. */ + static constexpr std::size_t node_alignment = alignof(Node); + +#ifdef PIXIE_SEQUENCE_TREE_TESTING + /** + * @brief Test-only transaction counters; absent without the testing macro. + * @details All translation units using an instantiation must agree on + * PIXIE_SEQUENCE_TREE_TESTING. Counts are thread-local per instantiation. + * Payload mutations count calls, not elements. Allocation counts include + * unused preflight spares; live counts include temporary/spare allocations. + */ + struct TestCounters { + std::size_t allocations = 0; + std::size_t node_visits = 0; + std::size_t child_transfers = 0; + std::size_t payload_mutations = 0; + std::size_t live_allocations = 0; + std::size_t peak_allocations = 0; + std::size_t live_bytes = 0; + std::size_t peak_bytes = 0; + }; + /** @brief Per-thread test counters, never present in normal builds. */ + inline static thread_local TestCounters test_counters{}; + /** + * @brief Fail allocation after n successful attempts; -1 disables injection. + * @param n Number of successful subsequent allocations before bad_alloc. + */ + static void test_fail_after(std::ptrdiff_t n) noexcept { failure_ = n; } + /** @brief Reset operation counters, retaining the current live allocation + * count. */ + static void test_reset_counters() noexcept { + const auto live = test_counters.live_allocations; + const auto bytes = test_counters.live_bytes; + test_counters = {}; + test_counters.live_allocations = test_counters.peak_allocations = live; + test_counters.live_bytes = test_counters.peak_bytes = bytes; + } +#endif + + private: + static void allocation() { +#ifdef PIXIE_SEQUENCE_TREE_TESTING + if (failure_ == 0) { + throw std::bad_alloc(); + } + if (failure_ > 0) { + --failure_; + } +#endif + } + template + static T* allocate() { + allocation(); + auto* p = new T; +#ifdef PIXIE_SEQUENCE_TREE_TESTING + ++test_counters.allocations; + ++test_counters.live_allocations; + test_counters.peak_allocations = std::max(test_counters.peak_allocations, + test_counters.live_allocations); + test_counters.live_bytes += sizeof(T); + test_counters.peak_bytes = + std::max(test_counters.peak_bytes, test_counters.live_bytes); +#endif + return p; + } + template + static void dispose(T* p) noexcept { + delete p; +#ifdef PIXIE_SEQUENCE_TREE_TESTING + --test_counters.live_allocations; + test_counters.live_bytes -= sizeof(T); +#endif + } + static void visit() noexcept { +#ifdef PIXIE_SEQUENCE_TREE_TESTING + ++test_counters.node_visits; +#endif + } + static void transfer() noexcept { +#ifdef PIXIE_SEQUENCE_TREE_TESTING + ++test_counters.child_transfers; +#endif + } + static void mutate() noexcept { +#ifdef PIXIE_SEQUENCE_TREE_TESTING + ++test_counters.payload_mutations; +#endif + } + static void destroy(void* p, std::size_t height) noexcept { + if (!p) { + return; + } + if (height == 0) { + dispose(static_cast(p)); + } else { + auto* node = static_cast(p); + for (std::size_t i = 0; i < node->count; ++i) { + destroy(node->children[i], height - 1); + } + dispose(node); + } + } + struct Owner { + void* p = nullptr; + std::size_t total = 0; + std::size_t height = 0; + Owner() noexcept = default; + Owner(void* pointer, std::size_t n, std::size_t h) noexcept + : p(pointer), total(n), height(h) {} + Owner(const Owner&) = delete; + Owner& operator=(const Owner&) = delete; + Owner(Owner&& other) noexcept + : p(std::exchange(other.p, nullptr)), + total(std::exchange(other.total, 0)), + height(std::exchange(other.height, 0)) {} + Owner& operator=(Owner&& other) noexcept { + if (this != &other) { + destroy(p, height); + p = std::exchange(other.p, nullptr); + total = std::exchange(other.total, 0); + height = std::exchange(other.height, 0); + } + return *this; + } + ~Owner() { destroy(p, height); } + void* release() noexcept { return std::exchange(p, nullptr); } + }; + + // Spares use their unoccupied first slot as a free-list link. No separate + // vector allocations, no live ownership objects copied as numeric arrays. + // Only this operation owns the pool; dismantled nodes join it after their + // children have transferred to Owners. Stale slots beyond count own nothing. + struct Pool { + Node* nodes = nullptr; + std::array leaves; + std::size_t leaf_count = 0; + Pool() = default; + Pool(const Pool&) = delete; + Pool& operator=(const Pool&) = delete; + ~Pool() { + while (nodes) { + auto* next = static_cast(nodes->children[0]); + dispose(nodes); + nodes = next; + } + } + void reserve(std::size_t node_count, std::size_t leaf_spares) { + for (std::size_t i = 0; i < node_count; ++i) { + recycle(allocate()); + } + for (; leaf_count < leaf_spares; ++leaf_count) { + leaves[leaf_count] = Owner(allocate(), 0, 0); + } + } + void recycle(Node* node) noexcept { + node->count = 0; + if constexpr (IndexBias) { + node->bias.value = 0; + } + node->children[0] = nodes; + nodes = node; + } + Node* node() noexcept { + assert(nodes); + assert(nodes->count == 0); + auto* result = nodes; + nodes = static_cast(nodes->children[0]); + result->children[0] = nullptr; + return result; + } + Owner leaf() noexcept { + assert(leaf_count); + return std::move(leaves[--leaf_count]); + } + }; + using Entries = std::array; + struct Pair { + Owner left, right; + }; + + static void add_bias(void* p, std::size_t height, std::uint64_t bias) noexcept + requires(IndexBias) + { + auto& pending = height == 0 ? static_cast(p)->bias.value + : static_cast(p)->bias.value; + assert(bias <= std::numeric_limits::max() - pending); + pending += bias; + } + static void normalize(Leaf& leaf) noexcept { + if constexpr (IndexBias) { + if (leaf.bias.value != 0) { + leaf.block.add_bias(leaf.bias.value); + leaf.bias.value = 0; + mutate(); + } + } + } + static void redistribute(Leaf& a, Leaf& b, std::size_t left) noexcept { + normalize(a); + normalize(b); + a.block.redistribute(b.block, left); + mutate(); + } + + static std::size_t length(const Node& node, + std::size_t i, + std::size_t total, + std::size_t prefix) noexcept { + if (i + 1 == node.count) { + return total - prefix; + } + if constexpr (Layout == LengthLayout::cumulative) { + return node.measures[i] - prefix; + } else { + return node.measures[i]; + } + } + static std::size_t unpack(Owner tree, + Entries& entries, + Pool& pool, + std::size_t offset = 0) noexcept { + visit(); + auto* node = static_cast(tree.p); + const auto count = node->count; + std::size_t prefix = 0; + for (std::size_t i = 0; i < count; ++i) { + const auto n = length(*node, i, tree.total, prefix); + if constexpr (IndexBias) { + add_bias(node->children[i], tree.height - 1, node->bias.value); + } + entries[offset + i] = Owner(node->children[i], n, tree.height - 1); + prefix += n; + transfer(); + } + tree.release(); + pool.recycle(node); + return count; + } + static Owner pack(Node* node, + Entries& entries, + std::size_t begin, + std::size_t count) noexcept { + assert(count >= 2 && count <= Fanout); + node->count = count; + const auto height = entries[begin].height + 1; + std::size_t total = 0; + for (std::size_t i = 0; i < count; ++i) { + auto& entry = entries[begin + i]; + assert(entry.p && entry.height + 1 == height); + total += entry.total; + node->children[i] = entry.release(); + if (i + 1 < count) { + node->measures[i] = + Layout == LengthLayout::cumulative ? total : entry.total; + } + transfer(); + } + return Owner(node, total, height); + } + static Owner group(Entries& entries, + std::size_t begin, + std::size_t count, + Pool& pool) noexcept { + if (count == 0) { + return {}; + } + if (count == 1) { + return std::move(entries[begin]); + } + return pack(pool.node(), entries, begin, count); + } + static Pair repack(Entries& entries, std::size_t count, Pool& pool) noexcept { + if (count <= Fanout) { + return {group(entries, 0, count, pool), {}}; + } + const auto half = count / 2; + return {group(entries, 0, half, pool), + group(entries, half, count - half, pool)}; + } + // Join compatible roots, carrying at most two nodes at the taller height. + // Underfull roots are merged/redistributed before becoming nonroot children. + // Pool deficit: equal-height unpack returns two nodes before repack uses at + // most two. Each of gap unequal levels returns one before using at most two; + // join may add one root. Thus gap+1 spares cover every prefix, not just the + // final node count. Without root growth the bound is gap. A taller root with + // a spare child slot cannot split, tightening this to gap-1 when gap > 0. + static Pair join_level(Owner a, Owner b, Pool& pool) noexcept { + if (a.height == b.height) { + if (a.height == 0) { + return {std::move(a), std::move(b)}; + } + Entries entries; + const auto n = unpack(std::move(a), entries, pool); + const auto m = unpack(std::move(b), entries, pool, n); + return repack(entries, n + m, pool); + } + Entries entries; + if (a.height > b.height) { + const auto count = unpack(std::move(a), entries, pool); + auto seam = join_level(std::move(entries[count - 1]), std::move(b), pool); + entries[count - 1] = std::move(seam.left); + const bool extra = seam.right.p != nullptr; + entries[count] = std::move(seam.right); + return repack(entries, count + extra, pool); + } + const auto count = unpack(std::move(b), entries, pool); + auto seam = join_level(std::move(a), std::move(entries[0]), pool); + const bool extra = seam.right.p != nullptr; + if (extra) { + for (std::size_t i = count; i > 1; --i) { + entries[i] = std::move(entries[i - 1]); + } + entries[1] = std::move(seam.right); + } + entries[0] = std::move(seam.left); + return repack(entries, count + extra, pool); + } + static Owner join(Owner a, Owner b, Pool& pool) noexcept { + if (!a.p) { + return b; + } + if (!b.p) { + return a; + } + auto pair = join_level(std::move(a), std::move(b), pool); + if (!pair.right.p) { + return std::move(pair.left); + } + Entries entries; + entries[0] = std::move(pair.left); + entries[1] = std::move(pair.right); + return group(entries, 0, 2, pool); + } + + // One cut descent. On ascent, sibling groups are already balanced subtrees. + // Each boundary accumulator rises monotonically: spine gaps traversed by + // join telescope across levels (plus one boundary node per level), O(F*h), + // not a fresh full-height join at every ancestor. Neither result exceeds the + // input height: sibling groups have at most F-1 children, so their root has + // room for a carry; a single sibling joined to a cut grows by at most one. + // + /** + * @brief Split with at most 3h fresh internal nodes, including prefix + * deficits. + * @details All d<=h cut ancestors are recycled before ascent. At a level + * with child height t, a nonempty boundary accumulator has height q<=t. + * One sibling costs at most t-q+1 join nodes. Two or more siblings cost one + * group node plus at most (t+1-q)-1 join nodes: the group has <=F-1 children, + * so its taller root cannot split. Their respective output heights are at + * least t and t+1. Thus charge each side's height increase plus one only + * for a singleton sibling. An empty accumulator costs at most one group, + * covered by its new height; no siblings cost nothing. Heights telescope + * to <=h per side, and there are <=2d singleton groups. Local prefix-deficit + * charges sum to <=2h+2d, less the d recycled ancestors: <=3h. Every prefix + * is covered, including both groups before either join and retained roots; + * grouping is prepaid by the same height-increase charge. It is not a bound + * inferred from final growth. Leaf cuts separately need one leaf spare. + */ + static Pair split(Owner tree, std::size_t p, Pool& pool) noexcept { + if (p == 0) { + return {{}, std::move(tree)}; + } + if (p == tree.total) { + return {std::move(tree), {}}; + } + if (tree.height == 0) { + auto right = pool.leaf(); + redistribute(*static_cast(tree.p), *static_cast(right.p), + p); + right.total = tree.total - p; + tree.total = p; + return {std::move(tree), std::move(right)}; + } + Entries entries; + const auto count = unpack(std::move(tree), entries, pool); + std::size_t i = 0; + while (p > entries[i].total) { + p -= entries[i++].total; + } + auto cut = split(std::move(entries[i]), p, pool); + auto left = group(entries, 0, i, pool); + auto right = group(entries, i + 1, count - i - 1, pool); + return {join(std::move(left), std::move(cut.left), pool), + join(std::move(cut.right), std::move(right), pool)}; + } + + struct Location { + Leaf* leaf; + std::size_t offset; + [[no_unique_address]] Bias bias{}; + }; + /** + * @brief Rotate within a leaf or reorder complete children at a covering + * node. + * @details Search is read-only until all three boundaries match child slots. + * Only global exterior leaves can be underfull. Check those included in a + * matched child range before committing; strict interior ranges need no leaf + * lookups. A descent reaching a leaf uses its nonthrowing local kernel. + * Typed pointer rotation transfers occupied-slot ownership; numeric lengths + * rotate separately and rebuild at most F-1 measures. Parent uniform bias + * stays in place and each child's own bias travels with its allocation. + * Ancestor totals, heights, and all allocation identities remain unchanged. + * @return True on nonallocating commit, false without any mutation otherwise. + */ + bool rotate_local(std::size_t left, + std::size_t right, + std::size_t distance) noexcept { + const bool includes_first = left == 0; + const bool includes_last = right == size(); + void* p = root_.p; + auto total = root_.total; + for (auto h = root_.height; h != 0; --h) { + visit(); + auto& node = *static_cast(p); + std::array lengths; + std::size_t prefix = 0; + std::size_t begin = Fanout, middle = Fanout, end = Fanout + 1; + bool descend = false; + for (std::size_t i = 0; i < node.count; ++i) { + const auto n = length(node, i, total, prefix); + if (left >= prefix && right <= prefix + n) { + left -= prefix; + right -= prefix; + total = n; + p = node.children[i]; + descend = true; + break; + } + lengths[i] = n; + if (prefix == left) { + begin = i; + } + if (prefix == left + distance) { + middle = i; + } + prefix += n; + if (prefix == right) { + end = i + 1; + } + } + if (descend) { + continue; + } + if (begin == Fanout || middle == Fanout || end == Fanout + 1) { + return false; + } + // The matched slots already identify the endpoint subtrees. Follow only + // their outer spines, without rescanning ancestors or selecting by rank. + const auto underfull = [h](void* child, bool back) noexcept { + for (auto level = h - 1; level != 0; --level) { + visit(); + const auto& edge = *static_cast(child); + child = edge.children[back ? edge.count - 1 : 0]; + } + return static_cast(child)->block.size() < minimum_leaf_size; + }; + if ((includes_first && underfull(node.children[begin], false)) || + (includes_last && underfull(node.children[end - 1], true))) { + return false; + } + std::rotate(node.children.begin() + begin, node.children.begin() + middle, + node.children.begin() + end); + std::rotate(lengths.begin() + begin, lengths.begin() + middle, + lengths.begin() + end); + prefix = 0; + for (std::size_t i = 0; i + 1 < node.count; ++i) { + prefix += lengths[i]; + node.measures[i] = + Layout == LengthLayout::cumulative ? prefix : lengths[i]; + } + for (auto i = begin; i < end; ++i) { + transfer(); + } + return true; + } + static_cast(p)->block.rotate_left(left, right, distance); + mutate(); + return true; + } + template + static Location locate(const Owner& root, std::size_t index) noexcept { + void* p = root.p; + auto total = root.total; + [[maybe_unused]] Bias bias{}; + for (auto h = root.height; h != 0; --h) { + visit(); + const auto& node = *static_cast(p); + if constexpr (Accumulate && IndexBias) { + bias.value += node.bias.value; + } + if constexpr (Layout == LengthLayout::cumulative) { + const auto i = + node_select({node.measures.data(), node.count - 1}, index); + const auto prefix = i == 0 ? 0 : node.measures[i - 1]; + const auto end = i + 1 == node.count ? total : node.measures[i]; + index -= prefix; + total = end - prefix; + p = node.children[i]; + } else { + std::size_t prefix = 0; + for (std::size_t i = 0; i < node.count; ++i) { + const auto n = length(node, i, total, prefix); + if (index - prefix < n) { + index -= prefix; + total = n; + p = node.children[i]; + break; + } + prefix += n; + } + } + } + if constexpr (Accumulate && IndexBias) { + bias.value += static_cast(p)->bias.value; + } + return {static_cast(p), index, bias}; + } + static Owner pop_edge(Owner& tree, bool back, Pool& pool) noexcept { + // Removing an entire exterior leaf needs no spares. Inductively, reducing + // a child from height k-1 to q returns at least k-1-q nodes. Recycling the + // parent adds one. With >=2 siblings, grouping uses one, but its root has + // <=F-1 children: joining costs at most k-q-1. With one sibling, no group + // node is needed and join costs <=k-q (one less without root growth). + // With none, all credits remain. Every prefix is funded and a height drop + // from k to r leaves at least k-r credits, completing the induction. + const auto n = locate(tree, back ? tree.total - 1 : 0).leaf->block.size(); + const auto position = back ? tree.total - n : n; + auto cut = split(std::move(tree), position, pool); + if (back) { + tree = std::move(cut.left); + return std::move(cut.right); + } + tree = std::move(cut.right); + return std::move(cut.left); + } + // Payload seam repair is performed once, not at each structural ancestor. + // At most two leaves from each side suffice: if anything remains outside, + // an extracted neighbor was interior and hence at least half full. Compact + // the four blocks, then balance the final pair. A lone underfull result can + // therefore only occur when it is an exterior leaf of the whole result. + // Four pop_edge calls never increase the node deficit. Group the <=4 seam + // leaves (4<=F) using <=1 node, then join twice, rather than once per leaf. + // For h>=1, the group has height <=1<=h. The joins cost <=h+1 and <=h+2 + // spares and produce height <=h+2. Thus 1+(h+1)+(h+2)=2h+4 covers every + // prefix, including retained results. For h=0 both inputs are extracted in + // full, so only the group can cost a node. No leaf spares are used. An + // underfull group root is repaired by join_level before becoming nonroot. + static Owner concatenate(Owner a, Owner b, Pool& pool) noexcept { + if (!a.p || !b.p) { + return join(std::move(a), std::move(b), pool); + } + if (locate(a, a.total - 1).leaf->block.size() >= minimum_leaf_size && + locate(b, 0).leaf->block.size() >= minimum_leaf_size) { + return join(std::move(a), std::move(b), pool); + } + Entries seam; + seam[1] = pop_edge(a, true, pool); + if (a.p) { + seam[0] = pop_edge(a, true, pool); + } + seam[2] = pop_edge(b, false, pool); + if (b.p) { + seam[3] = pop_edge(b, false, pool); + } + std::size_t count = 0; + for (std::size_t i = 0; i < 4; ++i) { + if (seam[i].p) { + if (i != count) { + seam[count] = std::move(seam[i]); + } + ++count; + } + } + for (std::size_t i = 0; i + 1 < count;) { + auto& x = seam[i]; + auto& y = seam[i + 1]; + const auto total = x.total + y.total; + const auto first = std::min(block_capacity, total); + redistribute(*static_cast(x.p), *static_cast(y.p), first); + x.total = first; + y.total = total - first; + if (y.total == 0) { + for (std::size_t j = i + 1; j + 1 < count; ++j) { + seam[j] = std::move(seam[j + 1]); + } + seam[--count] = {}; + } else { + ++i; + } + } + if (count >= 2 && seam[count - 1].total < minimum_leaf_size) { + auto& x = seam[count - 2]; + auto& y = seam[count - 1]; + const auto total = x.total + y.total; + redistribute(*static_cast(x.p), *static_cast(y.p), + total / 2); + x.total = total / 2; + y.total = total - x.total; + } + a = join(std::move(a), group(seam, 0, count, pool), pool); + return join(std::move(a), std::move(b), pool); + } + + public: + /** @brief Construct canonical empty, with no allocation. */ + SequenceTree() noexcept = default; + /** @brief Exclusive ownership disallows copying. */ + SequenceTree(const SequenceTree&) = delete; + /** @brief Exclusive ownership disallows copy assignment. */ + SequenceTree& operator=(const SequenceTree&) = delete; + /** @brief Transfer ownership; source becomes canonical empty. */ + SequenceTree(SequenceTree&&) noexcept = default; + /** @brief Transfer ownership; source becomes empty; self-move is a no-op. */ + SequenceTree& operator=(SequenceTree&&) noexcept = default; + /** @brief Return logical element count. @return Count in [0, SIZE_MAX]. */ + std::size_t size() const noexcept { return root_.total; } + /** @brief Test emptiness. @return Whether no elements or allocations exist. + */ + bool empty() const noexcept { return !root_.p; } + /** @brief Return internal levels. @return Zero for an empty tree or leaf + * root. */ + std::size_t height() const noexcept { return root_.height; } + /** + * @brief Read a zero-based element by value in O(F*h). + * @param i Index in [0, size()). + * @return Immutable logical value. + * @throws std::out_of_range If i >= size(). + */ + value_type operator[](std::size_t i) const { + if (i >= size()) { + throw std::out_of_range("SequenceTree: index"); + } + const auto position = locate(root_, i); + const auto value = std::as_const(position.leaf->block)[position.offset]; + if constexpr (IndexBias) { + return static_cast(value + position.bias.value); + } else { + return value; + } + } + /** + * @brief Consume a single-pass range of blocks in streaming bottom-up order. + * @details Moves from each input block; empty blocks are ignored. Packs short + * inputs into full leaves, retaining O(F*h) pending owners, not a directory + * or second complete payload buffer. Completed levels are built bottom-up; + * finishing the bounded fringe groups and joins once per level. On iterator, + * allocation or length failure, already visited input may be consumed; all + * acquired ownership is reclaimed. This is not a transactional range API. + * @param blocks Mutable input range whose references can move into Block. + * @return Exclusively owned tree preserving concatenated element order. + * @throws std::length_error If total exceeds SIZE_MAX or a block exceeds + * capacity. + * @throws std::bad_alloc If construction allocation fails. + */ + template + requires std:: + constructible_from> + static SequenceTree from_blocks(Range&& blocks) { + std::array levels; + std::array counts{}; + std::size_t total = 0; + Block pending; + auto emit = [&](Block&& block) { + Owner carry(allocate(), block.size(), 0); + static_cast(carry.p)->block = std::move(block); + std::size_t level = 0; + while (true) { + levels[level][counts[level]++] = std::move(carry); + if (counts[level] != Fanout) { + break; + } + auto* node = allocate(); + carry = pack(node, levels[level], 0, Fanout); + counts[level++] = 0; + assert(level < max_height); + } + }; + auto it = std::ranges::begin(blocks); + const auto end = std::ranges::end(blocks); + for (; it != end; ++it) { + Block incoming(std::ranges::iter_move(it)); + const auto n = incoming.size(); + if (n > Block::capacity || + n > std::numeric_limits::max() - total) { + throw std::length_error("SequenceTree: construction size"); + } + total += n; + if (n == 0) { + continue; + } + const auto combined = pending.size() + n; + pending.redistribute(incoming, std::min(block_capacity, combined)); + if (pending.size() == Block::capacity) { + emit(std::move(pending)); + pending = std::move(incoming); + } + } + if (pending.size()) { + emit(std::move(pending)); + } + SequenceTree result; + if (total <= Block::capacity) { + result.root_ = std::move(levels[0][0]); + return result; + } + // Assemble from low to high: each level's older sibling group precedes the + // existing fringe. Height gaps telescope just as in split's boundary + // assembly, rather than repeatedly descending from the tallest root. + Pool pool; + std::size_t used_height = 0; + for (std::size_t i = 0; i < max_height; ++i) { + if (counts[i]) { + used_height = i; + } + } + // <=h+1 groups and join roots, plus telescoping gaps <=h+1. This covers + // every prefix while the unassembled higher-level Owners remain live. + pool.reserve(3 * (used_height + 1), 0); + for (std::size_t i = 0; i <= used_height; ++i) { + auto prefix = group(levels[i], 0, counts[i], pool); + result.root_ = join(std::move(prefix), std::move(result.root_), pool); + } + return result; + } + /** + * @brief Keep [0,p), returning [p,size()) by exclusive ownership transfer. + * @details Endpoints are accepted. O(F*h) structure and at most one block + * redistribution. Allocation failure leaves this tree unchanged. + * @param p Zero-based split boundary in [0,size()]. + * @return Consumed suffix; neither result shares storage. + * @throws std::out_of_range If p > size(). + * @throws std::bad_alloc On preflight allocation failure. + */ + SequenceTree split_off(std::size_t p) { + if (p > size()) { + throw std::out_of_range("SequenceTree: split position"); + } + SequenceTree result; + if (p == size()) { + return result; + } + if (p == 0) { + return std::move(*this); + } + Pool pool; + pool.reserve(3 * root_.height, locate(root_, p).offset != 0); + auto pair = split(std::move(root_), p, pool); + root_ = std::move(pair.left); + result.root_ = std::move(pair.right); + return result; + } + /** + * @brief Append and consume a distinct donor in O(F*h). + * @details Self-merge and empty donor are no-ops. On success donor is + * canonical empty. Repairs only a bounded leaf seam; off-path ownership stays + * intact. Both trees remain unchanged on allocation or total-size failure. + * @param donor Tree whose elements follow this tree's contents. + * @throws std::length_error If concatenation exceeds SIZE_MAX. + * @throws std::bad_alloc On preflight allocation failure. + */ + void merge(SequenceTree& donor) { merge_impl(donor); } + + private: + // Only Permutation may request rebasing. All allocating preflight paths + // complete before tagging; from that point every ownership transfer and + // bounded normalization is noexcept. The structural pool proofs are + // unchanged. + void merge_rebased(SequenceTree& donor) + requires(IndexBias) + { + merge_impl(donor); + } + template + void merge_impl(SequenceTree& donor) { + if (this == &donor || donor.empty()) { + return; + } + if (donor.size() > std::numeric_limits::max() - size()) { + throw std::length_error("SequenceTree: concatenation size"); + } + if (empty()) { + *this = std::move(donor); + return; + } + Pool pool; + auto tag = [&]() noexcept { + if constexpr (Rebase) { + add_bias(donor.root_.p, donor.height(), size()); + } + }; + if (height() == 0 && donor.height() == 0) { + // Two leaf roots have no spine to rebuild. A fitting total combines in + // place; otherwise seam extraction splits only at endpoints and joining + // the two surviving leaves requires exactly one parent allocation. + const auto total = size() + donor.size(); + if (total <= block_capacity) { + tag(); + redistribute(*static_cast(root_.p), + *static_cast(donor.root_.p), total); + root_.total = total; + donor.root_ = {}; + } else { + pool.reserve(1, 0); + tag(); + root_ = concatenate(std::move(root_), std::move(donor.root_), pool); + } + return; + } + const auto h = std::max(height(), donor.height()); + if (std::as_const(locate(root_, size() - 1).leaf->block).size() >= + minimum_leaf_size && + std::as_const(locate(donor.root_, 0).leaf->block).size() >= + minimum_leaf_size) { + // Recycled spine nodes cover replacements; only gap+1 additional nodes + // can be simultaneously needed. No payload repair or leaf spares. + const auto gap = h - std::min(height(), donor.height()); + pool.reserve(gap + 1, 0); + tag(); + root_ = join(std::move(root_), std::move(donor.root_), pool); + return; + } + pool.reserve(2 * h + 4, 0); + tag(); + root_ = concatenate(std::move(root_), std::move(donor.root_), pool); + } + + public: + /** + * @brief Rotate [left,right) left, preserving all outside elements. + * @details A one-leaf range uses its nonthrowing local kernel. Complete-child + * ranges at a common covering node reorder in place when both endpoint leaves + * are at least half full. Otherwise one preflight covers cuts and joins, so + * no intermediate edit can escape on allocation failure. O(F*h) structure, + * bounded leaf copying; no persistent spare-node storage. + * @param left Inclusive start. + * @param right Exclusive end. + * @param distance Left distance reduced modulo a nonempty range length. + * @throws std::out_of_range If left > right or right > size(), even at + * distance zero. + * @throws std::bad_alloc On preflight failure, leaving contents unchanged. + */ + void rotate_left(std::size_t left, std::size_t right, std::size_t distance) { + if (left > right || right > size()) { + throw std::out_of_range("SequenceTree: rotation range"); + } + const auto n = right - left; + if (n == 0 || (distance %= n) == 0) { + return; + } + if (rotate_local(left, right, distance)) { + return; + } + Pool pool; + if (left == 0 && right == size()) { + /** @brief One split (3h) and one seam (2h+4), with cut heights <=h. */ + pool.reserve(5 * height() + 4, locate(root_, distance).offset != 0); + auto cut = split(std::move(root_), distance, pool); + root_ = concatenate(std::move(cut.right), std::move(cut.left), pool); + return; + } + /** + * @brief Reserve the sum of cut and seam temporary-prefix deficits. + * @details Three splits cost <=9h, retaining all four results. Three seams + * cost <=(2h+4)+(2(h+2)+4)+(2(h+4)+4), each growing height by at most two. + * The sum is 15h+24. Only nonaligned cuts need leaf spares; earlier cuts + * preserve later offsets. This is not a net-growth-only reservation. + */ + const auto leaf_spares = + std::size_t(left != 0 && locate(root_, left).offset != 0) + + (locate(root_, left + distance).offset != 0) + + (right != size() && locate(root_, right).offset != 0); + pool.reserve(15 * height() + 24, leaf_spares); + auto suffix = split(std::move(root_), right, pool); + auto b = split(std::move(suffix.left), left + distance, pool); + auto a = split(std::move(b.left), left, pool); + root_ = concatenate(std::move(a.left), std::move(b.right), pool); + root_ = concatenate(std::move(root_), std::move(a.right), pool); + root_ = concatenate(std::move(root_), std::move(suffix.right), pool); + } + + /** @brief Explicit traversal result for memory accounting, excluding + * allocator overhead. */ + struct MemoryUsage { + std::size_t blocks = 0; + std::size_t nodes = 0; + std::size_t block_bytes = 0; + std::size_t node_bytes = 0; + std::size_t total_bytes = sizeof(SequenceTree); + }; + /** + * @brief Enumerate allocations for diagnostics, never called by mutation. + * @details O(number of allocations). Counts actual typed object sizes, + * including alignment padding, but excludes allocator overhead and preflight + * peak. Byte arithmetic saturates at SIZE_MAX if an exotic block represents + * too many tiny elements for the diagnostic footprint to be representable. + * @return Live block/node counts and requested bytes including this object. + */ + MemoryUsage memory_usage() const noexcept { + MemoryUsage result; + auto walk = [&](auto&& self, void* p, std::size_t h) -> void { + if (!p) { + return; + } + if (h == 0) { + ++result.blocks; + } else { + ++result.nodes; + const auto& node = *static_cast(p); + for (std::size_t i = 0; i < node.count; ++i) { + self(self, node.children[i], h - 1); + } + } + }; + walk(walk, root_.p, height()); + result.block_bytes = saturated_multiply(result.blocks, sizeof(Leaf)); + result.node_bytes = saturated_multiply(result.nodes, sizeof(Node)); + result.total_bytes = saturated_add( + sizeof(*this), saturated_add(result.block_bytes, result.node_bytes)); + return result; + } + /** @brief Explicit O(allocations) footprint query. @return Requested live + * bytes. */ + std::size_t memory_usage_bytes() const noexcept { + return memory_usage().total_bytes; + } + /** @brief Explicit O(allocations) leaf count. @return Number of live blocks. + */ + std::size_t block_count() const noexcept { return memory_usage().blocks; } + /** + * @brief Explicit O(allocations) internal-node count. + * @return Number of live internal nodes, excluding leaves and the root owner. + */ + std::size_t node_count() const noexcept { return memory_usage().nodes; } + /** @brief Explicit O(allocations) leaf bytes. @return Includes leaf + * metadata/padding. */ + std::size_t block_allocation_bytes() const noexcept { + return memory_usage().block_bytes; + } + /** @brief Explicit O(allocations) node bytes. @return Excludes the root + * owner. */ + std::size_t internal_memory_bytes() const noexcept { + return memory_usage().node_bytes; + } + /** + * @brief Explicit O(allocations) payload accounting for blocks reporting it. + * @return Payload capacity bytes including unused elements, excluding + * metadata. + */ + std::size_t payload_capacity_bytes() const noexcept + requires requires(const Block& b) { + { b.payload_capacity_bytes() } noexcept -> std::same_as; + } + { + return saturated_multiply(block_count(), Block{}.payload_capacity_bytes()); + } + /** + * @brief Explicit O(allocations) metadata accounting for payload-reporting + * blocks. + * @return Root owner, internal nodes, block metadata and alignment padding. + */ + std::size_t metadata_bytes() const noexcept + requires requires(const Block& b) { + { b.payload_capacity_bytes() } noexcept -> std::same_as; + } + { + return memory_usage_bytes() - payload_capacity_bytes(); + } + /** + * @brief Visit blocks in logical order without materializing another buffer. + * @details Explicit O(allocations) traversal; callback receives const Block&. + * Mutation during traversal is unsupported; callback exceptions propagate. + * Unavailable on tagged trees: raw stored fields omit pending index biases. + * @param callback Callable invoked once per nonempty block in sequence order. + */ + template + void for_each_block(Callback&& callback) const + requires(!IndexBias) + { + auto walk = [&](auto&& self, void* p, std::size_t h) -> void { + if (!p) { + return; + } + if (h == 0) { + callback(std::as_const(static_cast(p)->block)); + } else { + const auto& node = *static_cast(p); + for (std::size_t i = 0; i < node.count; ++i) { + self(self, node.children[i], h - 1); + } + } + }; + walk(walk, root_.p, height()); + } + +#ifdef PIXIE_SEQUENCE_TREE_TESTING + /** + * @brief Enumerate absolute child boundaries for shape-forcing rotation + * tests. + * @details Root-first preorder; each vector starts at the node's first + * element and ends at its exclusive end. Read-only O(allocations) traversal, + * absent from normal builds, with no effect on transaction counters or lazy + * biases. + * @return One allocating boundary vector per internal node. + */ + std::vector> test_child_boundaries() const { + std::vector> result; + auto walk = [&](auto&& self, const void* p, std::size_t total, + std::size_t h, std::size_t start) -> void { + if (!p || h == 0) { + return; + } + const auto& node = *static_cast(p); + std::vector boundaries{start}; + std::size_t prefix = 0; + for (std::size_t i = 0; i < node.count; ++i) { + prefix += length(node, i, total, prefix); + boundaries.push_back(start + prefix); + } + result.push_back(boundaries); + for (std::size_t i = 0; i < node.count; ++i) { + self(self, node.children[i], boundaries[i + 1] - boundaries[i], h - 1, + boundaries[i]); + } + }; + walk(walk, root_.p, root_.total, root_.height, 0); + return result; + } + + /** + * @brief Test-only recursive validation; never invoked by normal operations. + * @details Checks lengths, depth, node/leaf occupancy, unique ownership and + * allocation alignment. O(allocations) space/time, may allocate and throw. + * @return Whether every representation invariant holds. + */ + bool test_validate() const { + if (!root_.p) { + return size() == 0 && height() == 0; + } + std::unordered_set seen; + auto walk = [&](auto&& self, void* p, std::size_t total, std::size_t h, + bool first, bool last, bool root, + std::uint64_t inherited) -> bool { + if (!p || !total || !seen.insert(p).second) { + return false; + } + if (h == 0) { + const auto& b = static_cast(p)->block; + if constexpr (IndexBias) { + const auto pending = static_cast(p)->bias.value; + const auto maximum = std::numeric_limits::max(); + if (inherited > maximum || pending > maximum - inherited) { + return false; + } + inherited += pending; + for (std::size_t i = 0; i < b.size(); ++i) { + if (b[i] > maximum - inherited) { + return false; + } + } + } + return reinterpret_cast(p) % alignof(Leaf) == 0 && + b.size() == total && total <= Block::capacity && + (first || last || total >= minimum_leaf_size); + } + const auto& node = *static_cast(p); + if constexpr (IndexBias) { + const auto maximum = std::numeric_limits::max(); + if (inherited > maximum || node.bias.value > maximum - inherited) { + return false; + } + inherited += node.bias.value; + } + if (reinterpret_cast(p) % alignof(Node) != 0 || + node.count < (root ? 2 : Fanout / 2) || node.count > Fanout) { + return false; + } + std::size_t prefix = 0; + for (std::size_t i = 0; i < node.count; ++i) { + if (prefix >= total) { + return false; + } + const auto n = length(node, i, total, prefix); + if (n > total - prefix || + !self(self, node.children[i], n, h - 1, first && i == 0, + last && i + 1 == node.count, false, inherited)) { + return false; + } + prefix += n; + } + return prefix == total; + }; + return walk(walk, root_.p, size(), height(), true, true, true, 0); + } + /** @brief Enumerate stable leaf identities for locality tests. @return + * Ordered addresses. */ + std::vector test_leaf_identities() const { + std::vector result; + auto walk = [&](auto&& self, const void* p, std::size_t h) -> void { + if (!p) { + return; + } + if (h == 0) { + result.push_back(&static_cast(p)->block); + } else { + const auto& node = *static_cast(p); + for (std::size_t i = 0; i < node.count; ++i) { + self(self, node.children[i], h - 1); + } + } + }; + walk(walk, root_.p, height()); + return result; + } + /** + * @brief Snapshot raw pending biases and their allocation identities in + * tests. + * @details Tagged trees only; root-first preorder includes every node and + * leaf, including zero biases. This const traversal neither pushes tags nor + * changes operation counters. O(allocations) time and returned vector space; + * vector allocation may throw. Absent from production builds. + * @return Allocation address and exact unaccumulated pending bias pairs. + */ + std::vector> test_bias_snapshot() const + requires(IndexBias) + { + std::vector> result; + auto walk = [&](auto&& self, const void* p, std::size_t h) -> void { + if (!p) { + return; + } + if (h == 0) { + result.emplace_back(p, static_cast(p)->bias.value); + } else { + const auto& node = *static_cast(p); + result.emplace_back(p, node.bias.value); + for (std::size_t i = 0; i < node.count; ++i) { + self(self, node.children[i], h - 1); + } + } + }; + walk(walk, root_.p, height()); + return result; + } + /** + * @brief Attach a synthetic test-only bias without huge identity allocation. + * @details Requires a nonempty tree and every resulting value representable. + * Not present in production and never an unchecked public container action. + */ + void test_add_bias(std::uint64_t bias) noexcept + requires(IndexBias) + { + assert(!empty()); + add_bias(root_.p, height(), bias); + } + /** + * @brief Enumerate internal-node identities for test-only locality checks. + * @details Explicit O(allocations) traversal, never called by mutation. + * Addresses follow root-first preorder with children in logical order. + * The returned vector allocates; callback-free enumeration does not update + * operation counters. Empty trees and leaf roots return an empty vector. + * @return Internal allocation addresses, excluding leaves and the root owner. + */ + std::vector test_internal_node_identities() const { + std::vector result; + auto walk = [&](auto&& self, const void* p, std::size_t h) -> void { + if (!p || h == 0) { + return; + } + result.push_back(p); + const auto& node = *static_cast(p); + for (std::size_t i = 0; i < node.count; ++i) { + self(self, node.children[i], h - 1); + } + }; + walk(walk, root_.p, height()); + return result; + } +#endif + + private: + static std::size_t saturated_add(std::size_t a, std::size_t b) noexcept { + return b > std::numeric_limits::max() - a + ? std::numeric_limits::max() + : a + b; + } + static std::size_t saturated_multiply(std::size_t a, std::size_t b) noexcept { + return b != 0 && a > std::numeric_limits::max() / b + ? std::numeric_limits::max() + : a * b; + } +#ifdef PIXIE_SEQUENCE_TREE_TESTING + inline static thread_local std::ptrdiff_t failure_ = -1; +#endif + Owner root_; +}; + +} // namespace pixie::detail::sequence +/// @endcond diff --git a/include/pixie/detail/sequence/sequence_tree_kernels.h b/include/pixie/detail/sequence/sequence_tree_kernels.h new file mode 100644 index 0000000..5f7be59 --- /dev/null +++ b/include/pixie/detail/sequence/sequence_tree_kernels.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include +#include + +#if SIZE_MAX == UINT64_MAX && (defined(__AVX512F__) || defined(__AVX2__)) +#include +#endif + +/// @cond PIXIE_SEQUENCE_INTERNAL +namespace pixie::detail::sequence { + +/** + * @brief Scalar child selection from unsigned cumulative exclusive ends. + * @details Linear reference and fallback for node_select; retains no storage. + * @param ends Nondecreasing stored ends, excluding the implicit last child end. + * The span must remain valid for this call; no alignment or padding is + * required. + * @param index Zero-based position to locate, with the full size_t range + * accepted. + * @return First position whose end is strictly greater than index, or + * ends.size() for the implicit last child. Equivalently, the number of ends <= + * index. An empty span, including null backing storage, returns zero without + * access. + * @pre ends is nondecreasing; violating this precondition is unsupported. + */ +inline std::size_t node_select_scalar(std::span ends, + std::size_t index) noexcept { + std::size_t i = 0; + while (i < ends.size() && ends[i] <= index) { + ++i; + } + return i; +} + +/** + * @brief Compile-time SIMD dispatch for node_select_scalar's exact contract. + * @param ends Nondecreasing exclusive ends, omitting the implicit last child. + * Only [0, ends.size()) is read; the span needs no alignment or extra padding. + * @param index Unsigned zero-based position, including values with the top bit + * set. + * @return First end > index, or ends.size() if none; zero without memory access + * for an empty span. Equality selects the following child. + * @pre ends remains valid during the call and is nondecreasing. + * @details Uses only AVX-512F, otherwise AVX2, when size_t is 64 bits; all + * other configurations use the scalar control. No backing storage is retained. + */ +inline std::size_t node_select(std::span ends, + std::size_t index) noexcept { + if (ends.empty()) { + return 0; + } +#if SIZE_MAX == UINT64_MAX && defined(__AVX512F__) + const auto query = + _mm512_set1_epi64(std::bit_cast(std::uint64_t{index})); + for (std::size_t i = 0; i < ends.size();) { + const auto remaining = ends.size() - i; + const auto count = remaining < 8 ? remaining : 8; + const auto active = static_cast<__mmask8>((1u << count) - 1); + const auto values = _mm512_maskz_loadu_epi64(active, ends.data() + i); + const auto greater = static_cast( + _mm512_mask_cmp_epu64_mask(active, values, query, _MM_CMPINT_GT)); + if (greater != 0) { + return i + std::countr_zero(greater); + } + i += count; + } + return ends.size(); +#elif SIZE_MAX == UINT64_MAX && defined(__AVX2__) + const auto sign = _mm256_set1_epi64x(std::numeric_limits::min()); + const auto query = _mm256_xor_si256( + _mm256_set1_epi64x(std::bit_cast(std::uint64_t{index})), sign); + std::size_t i = 0; + for (; ends.size() - i >= 4; i += 4) { + const auto values = + _mm256_loadu_si256(reinterpret_cast(ends.data() + i)); + const auto greater = + static_cast(_mm256_movemask_pd(_mm256_castsi256_pd( + _mm256_cmpgt_epi64(_mm256_xor_si256(values, sign), query)))); + if (greater != 0) { + return i + std::countr_zero(greater); + } + } + return i + node_select_scalar(ends.subspan(i), index); +#else + return node_select_scalar(ends, index); +#endif +} + +} // namespace pixie::detail::sequence +/// @endcond diff --git a/include/pixie/experimental/permuted_bit_block.h b/include/pixie/experimental/permuted_bit_block.h new file mode 100644 index 0000000..e5e9860 --- /dev/null +++ b/include/pixie/experimental/permuted_bit_block.h @@ -0,0 +1,284 @@ +#pragma once + +// Current experiment snapshot, 2026-09-16. Ryzen 7 8845HS, Linux/WSL, +// GCC 13.3, -O3 -DNDEBUG -march=native, Google Benchmark 1.9.4, CPU 0. +// Command: cpp-bench native bit_sequence_benchmarks 5 0.15s +// Filter: '^Block(Rotate|Read)<(DirectBlock|MappedBlock).*' +// Median CPU ns/op; randomized repetition interleaving. Hot single block, +// 256 precomputed rotations (seed 128), no construction/reset in timing. +// Modes 0/1: chunk-aligned/unaligned proper subranges; 2: alternating 50/50; +// 3: whole block by 65; 4: unaligned, N=2045, initial h=2038 (both variants +// retain their circular origin). Modes 1/4 retain identity maps; mode 2 keeps +// changing nonidentity maps across partial rotations without normalization. +// Read setup: two aligned subrange rotations, then whole-block rotation by 65; +// indexed reads use 1024 precomputed positions (seed 123). Flatten includes +// producing all 32 output words with zero padding, without mutating the block. +// +// | Workload | N bits | Direct ns | Mapped ns | +// | ---------------- | -----: | --------: | --------: | +// | Rotate aligned | 2048 | 16.85 | 3.82 | +// | Rotate unaligned | 2048 | 28.15 | 29.70 | +// | Rotate mixed | 2048 | 22.03 | 48.95 | +// | Rotate whole | 2048 | 3.29 | 3.29 | +// | Rotate wrapped | 2045 | 29.65 | 31.11 | +// | Indexed access | 2048 | 1.50 | 1.58 | +// | Indexed access | 2045 | 1.50 | 1.56 | +// | Flatten | 2048 | 10.57 | 99.69 | +// | Flatten | 2045 | 11.36 | 99.55 | +// +// Direct/Mapped use the same 256-byte payload and packed copy primitive. +// Object bytes: Direct 272, Mapped 280. Map adds 8 bytes (3.125% of payload). +// Retain only as a bounded aligned-heavy experiment: mixed rotations and +// flattening remain substantially slower than direct. These are block-only +// results, not tree measurements. Chunk-wise gather/scatter avoids +// normalization, not map cost. Independent hot indexed reads measure +// throughput, not dependent latency; WSL timings do not establish cold-cache or +// large-sequence performance. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie::experimental { + +/** + * @brief Bounded owning experiment, independent of sequence split/merge policy. + * + * @details Holds 0..2048 bits in 16 physical 128-bit chunks. With + * Permuted=true, nibble j of a uint64_t maps canonical chunk j to its physical + * chunk. Circular origin h is applied modulo the valid length BEFORE this + * mapping. Whole-block rotation changes only h. At h=0, chunk-aligned subranges + * rotate map nibbles without touching payload (including in partially filled + * blocks). Other rotations gather only the affected interval in rotated order, + * then scatter through the same map, preserving both the permutation and + * origin. Copies translate contiguous chunk pieces, never individual bits. + * + * Permuted=false is the matched direct-movement control: the same packed copy + * primitive and circular-origin fast path, without a map or descriptor lookup. + * Value copies own independent payloads. No allocation or external lifetime. + * @tparam Permuted Whether to map canonical chunks to physical chunks; false + * selects the direct-movement control. + */ +template +class PermutedBitBlock { + public: + /** @brief Logical element type returned by value. */ + using value_type = bool; + /** @brief Maximum logical element count. */ + static constexpr std::size_t capacity = 2048; + /** @brief Fixed owning payload, in least-significant-bit-first word order. */ + using Payload = std::array; + + /** @brief Construct empty; all storage is initialized. */ + PermutedBitBlock() noexcept = default; + + /** + * @brief Copy n bits, ignoring extra words and high padding bits. + * @details The source need not outlive the block. Bits are copied in + * least-significant-bit-first word order, with initial circular origin h = 0. + * @param words Source words containing the bits to copy. + * @param n Valid logical bit count, from 0 through 2048 inclusive. + * @throws std::invalid_argument If n > 2048 or words does not cover n bits. + */ + PermutedBitBlock(std::span words, std::size_t n) + : n_(n) { + if (n > 2048 || (n + 63) / 64 > words.size()) { + throw std::invalid_argument("PermutedBitBlock: input size"); + } + std::copy_n(words.begin(), (n + 63) / 64, bits_.begin()); + } + + /** + * @brief Query the logical length. + * @return Valid logical bit count, excluding capacity and padding. + */ + std::size_t size() const noexcept { return n_; } + /** + * @brief Query whether the block is empty. + * @return True if there are no valid bits. + */ + bool empty() const noexcept { return n_ == 0; } + + /** + * @brief Read a zero-based logical bit by value. + * @param i Bit position in [0, size()). + * @return The logical bit value. + * @throws std::out_of_range If i >= size(). + */ + bool operator[](std::size_t i) const { + if (i >= n_) { + throw std::out_of_range("PermutedBitBlock: index"); + } + const auto p = physical((h_ + i) % n_); + return (bits_[p / 64] >> (p % 64)) & 1; + } + + /** + * @brief Copy logical bits to linear words without mutating the block. + * @details Copies at most 256 bytes and does not allocate. + * @return Owning LSB-first payload with all bits beyond size() set to zero. + */ + Payload flatten() const noexcept { + Payload result{}; + copy(0, result, 0, n_); + return result; + } + + /** + * @brief Rotate [left, right) left by distance modulo its length. + * @details Empty ranges and zero effective distances do nothing. Bits outside + * the range remain unchanged. + * @param left Inclusive zero-based start of the range. + * @param right Exclusive end of the range. + * @param distance Left rotation distance in bits, reduced modulo range + * length. + * @pre left <= right <= size(); invalid primitive arguments are asserted. + */ + void rotate_left(std::size_t left, + std::size_t right, + std::size_t distance) noexcept { + assert(left <= right && right <= n_); + const auto length = right - left; + if (length == 0 || (distance %= length) == 0) { + return; + } + if (length == n_) { + h_ = (h_ + distance) % n_; + return; + } + if constexpr (Permuted) { + if (h_ == 0 && ((left | right | distance) % 128 == 0)) { + const auto shift = left / 128 * 4; + const auto width = length / 128 * 4; + const auto cut = distance / 128 * 4; + // Proper subrange: width < 64; distance is strictly inside the range. + const auto mask = (std::uint64_t{1} << width) - 1; + const auto slice = (map_ >> shift) & mask; + const auto rotated = (slice >> cut) | (slice << (width - cut)); + map_ = (map_ & ~(mask << shift)) | ((rotated & mask) << shift); + return; + } + } + Payload scratch{}; + copy(left + distance, scratch, 0, length - distance); + copy(left, scratch, length - distance, distance); + const auto position = (h_ + left) % n_; + if constexpr (Permuted) { + if (map_ != identity) { + auto p = position; + std::size_t source = 0; + while (source != length) { + const auto run = std::min({length - source, n_ - p, 128 - p % 128}); + copy_packed_bits(scratch.data(), source, bits_.data(), physical(p), + run); + source += run; + p = (p + run == n_) ? 0 : p + run; + } + return; + } + } + const auto prefix = std::min(length, n_ - position); + copy_packed_bits(scratch.data(), 0, bits_.data(), position, prefix); + copy_packed_bits(scratch.data(), prefix, bits_.data(), 0, length - prefix); + } + + /** + * @brief Query fixed payload capacity. + * @return Payload bytes, including unused capacity. + */ + std::size_t payload_capacity_bytes() const noexcept { return sizeof(bits_); } + /** + * @brief Query object metadata size; there is no dynamic allocation. + * @return Object bytes excluding the payload, including any padding. + */ + std::size_t metadata_bytes() const noexcept { + return sizeof(*this) - sizeof(bits_); + } + + /** + * @brief Repartition two distinct blocks without allocation or exceptions. + * @details Preserves concatenated logical contents; length changes normalize + * the maps and origins. Ordinary rotations retain their existing kernels. + * Requires left_size <= capacity and left_size <= size()+rhs.size() <= + * left_size+capacity. Invalid primitive arguments are asserted. + * @param rhs Right block, receiving the suffix. + * @param left_size Desired valid count in this block. + */ + void redistribute(PermutedBitBlock& rhs, std::size_t left_size) noexcept { + assert(this != &rhs); + const auto total = n_ + rhs.n_; + assert(left_size <= capacity && left_size <= total && + total - left_size <= capacity); + Payload a{}, b{}; + auto gather = [&](std::size_t start, Payload& out, std::size_t count) { + const auto first = start < n_ ? std::min(count, n_ - start) : 0; + copy(start, out, 0, first); + rhs.copy(start + first - std::min(start + first, n_), out, first, + count - first); + }; + gather(0, a, left_size); + gather(left_size, b, total - left_size); + bits_ = a; + rhs.bits_ = b; + n_ = left_size; + rhs.n_ = total - left_size; + h_ = rhs.h_ = 0; + if constexpr (Permuted) { + map_ = rhs.map_ = identity; + } + } + + private: + static constexpr std::uint64_t identity = 0xfedcba9876543210ULL; + struct NoMap {}; + + std::size_t physical(std::size_t p) const noexcept { + if constexpr (Permuted) { + return ((map_ >> (p / 128 * 4)) & 15) * 128 + p % 128; + } + return p; + } + + void copy(std::size_t offset, + Payload& out, + std::size_t dest, + std::size_t count) const noexcept { + if (count == 0) { + return; + } + auto p = (h_ + offset) % n_; + while (count != 0) { + auto run = std::min(count, n_ - p); + if constexpr (Permuted) { + if (map_ != identity) { + run = std::min(run, 128 - p % 128); + } + } + copy_packed_bits(bits_.data(), physical(p), out.data(), dest, run); + count -= run; + dest += run; + p = (p + run == n_) ? 0 : p + run; + } + } + + Payload bits_{}; + std::size_t n_ = 0; + std::size_t h_ = 0; + [[no_unique_address]] std::conditional_t + map_ = []() noexcept { + if constexpr (Permuted) { + return identity; + } else { + return NoMap{}; + } + }(); +}; + +} // namespace pixie::experimental diff --git a/include/pixie/permutable_sequence.h b/include/pixie/permutable_sequence.h new file mode 100644 index 0000000..bf31fb3 --- /dev/null +++ b/include/pixie/permutable_sequence.h @@ -0,0 +1,148 @@ +#pragma once + +/** + * @file permutable_sequence.h + * @brief Lightweight immutable-read permutable-sequence CRTP contract. + * @details Include pixie/permutations/sequence.h for the owning implementation. + */ + +#include + +#include +#include +#include +#include + +namespace pixie { +/** + * @brief Compile-time element storage selection. + * @details Automatic packs bool and unsigned integers of at most 64 value bits; + * all other supported types use stable indirect ownership. Indirect may be + * requested explicitly for otherwise packable types. + */ +enum class ElementStorage { automatic, packed, indirect }; + +/** + * @brief CRTP contract for exclusively owned sequences with immutable reads. + * @details Merge appends unchanged values, unlike permutation rebasing. + * Implementations provide constrained from_range_impl(range), size_impl(), + * value_at_impl(i), rotate_left_impl(left,right,distance), merge_impl(donor), + * and memory_usage_bytes_impl() with the facade semantics documented below. + * size_impl() and memory_usage_bytes_impl() must be noexcept. Other extension + * points propagate the documented exceptions. Validation belongs to the + * implementation; the facade does not repeat it. No virtual dispatch. + * Required signatures are size_type size_impl() const noexcept, + * ConstReference value_at_impl(size_type) const, + * void rotate_left_impl(size_type,size_type,size_type), void merge_impl(Impl&), + * and size_type memory_usage_bytes_impl() const noexcept. The static factory + * returns Impl and accepts Range&& with the same input-range and iterator-move + * constructibility constraints as from_range(). Private extension points grant + * friendship to this base. Concrete owners default-construct canonical empty + * and are move-only with nonthrowing move and destruction; moves leave the + * source canonical empty and self-move is a no-op. + * @tparam Impl Concrete owning implementation. + * @tparam T Stored object type. + * @tparam ConstReference Explicit result type: T or const T&, avoiding lookup + * in the incomplete CRTP implementation and preventing reference copies. + */ +template + requires(std::same_as || + std::same_as) +class PermutableSequenceBase { + public: + /** @brief Stored element type. @details Independent of representation. */ + using value_type = T; + /** @brief Immutable result type. @details T by value or stable const T&. */ + using const_reference = ConstReference; + /** @brief Count and position type. @details Full unsigned size_t range. */ + using size_type = std::size_t; + /** + * @brief Consume a possibly single-pass input range into an owning sequence. + * @details from_range_impl consumes through ranges::iter_move, preserving + * order and supporting non-common, unsized ranges. On allocation, iteration, + * or element-construction failure all acquired ownership is reclaimed, but + * visited inputs may already be consumed. Input references are not retained. + * This is not the strong mutation guarantee for existing containers. + * @tparam Range Input range whose iterator-move results construct T. + * @param range Source consumed in iteration order. + * @return Exclusive owner of the input values in order. + * @throws std::length_error If count or configured chunk capacity is too + * large. + * @throws std::bad_alloc If storage allocation fails. + * @throws Any Exception from input iteration or element construction. + */ + template + requires std:: + constructible_from> + static Impl from_range(Range&& range) { + return Impl::from_range_impl(std::forward(range)); + } + /** + * @brief Return the logical element count. + * @details size_impl() reads the count without allocation or mutation. + * @return Count in [0,SIZE_MAX], measured in elements. + */ + size_type size() const noexcept { return impl().size_impl(); } + /** + * @brief Test emptiness. + * @details Equivalent to size()==0; canonical empty owns no heap storage. + * @return Whether no elements remain. + */ + bool empty() const noexcept { return size() == 0; } + /** + * @brief Read a checked zero-based element without exposing mutation. + * @details value_at_impl(position) returns exactly const_reference. Indirect + * references survive rotation, merge (including consumed donor references), + * and ownership moves. They expire when the eventual owner destroys or + * replaces their payload. No contiguous buffer or sentinel is promised. + * @param position Position in [0,size()). + * @return Immutable value or stable reference, according to storage policy. + * @throws std::out_of_range If position >= size(). + */ + const_reference operator[](size_type position) const { + return impl().value_at_impl(position); + } + /** + * @brief Rotate [left,right) left while preserving outside values. + * @details rotate_left_impl validates even at zero distance; empty intervals + * are no-ops, otherwise distance is reduced modulo right-left. Allocation + * failure leaves contents and indirect addresses unchanged. + * @param left Inclusive start. + * @param right Exclusive end, at most size(). + * @param distance Left rotation distance in elements. + * @throws std::out_of_range If left > right or right > size(). + * @throws std::bad_alloc If preflight allocation fails. + */ + void rotate_left(size_type left, size_type right, size_type distance) { + impl().rotate_left_impl(left, right, distance); + } + /** + * @brief Append unchanged donor values and consume donor ownership. + * @details merge_impl accepts only identical configurations. Self merge and + * empty donor are no-ops; success leaves donor canonical empty. Count or + * allocation failure leaves both containers unchanged. Indirect objects + * never move and references from either participant retain their addresses. + * @param donor Owner whose unchanged values follow this sequence's values. + * @throws std::length_error If the combined count exceeds SIZE_MAX. + * @throws std::bad_alloc If preflight allocation fails. + */ + void merge(Impl& donor) { impl().merge_impl(donor); } + /** + * @brief Explicitly account for requested live storage. + * @details memory_usage_bytes_impl() traverses allocations and chunks in + * linear time without allocating or throwing. Includes facade, metadata, + * padding, and payload capacity including slack; saturates at SIZE_MAX. + * Excludes allocations inside T, allocator overhead, RSS, stack scratch and + * temporary peaks. Concrete memory_usage() has subset fields that must not + * be added again to its disjoint total categories. + * @return Requested live bytes including this owner exactly once. + */ + size_type memory_usage_bytes() const noexcept { + return impl().memory_usage_bytes_impl(); + } + + private: + Impl& impl() noexcept { return static_cast(*this); } + const Impl& impl() const noexcept { return static_cast(*this); } +}; +} // namespace pixie diff --git a/include/pixie/permutation.h b/include/pixie/permutation.h new file mode 100644 index 0000000..58b8189 --- /dev/null +++ b/include/pixie/permutation.h @@ -0,0 +1,129 @@ +#pragma once + +/** + * @file permutation.h + * @brief Lightweight permutation CRTP contract. + * @details Include pixie/permutations/permutation.h for the owning + * implementation. + */ + +#include + +#include +#include +#include +#include + +namespace pixie { +/** + * @brief CRTP contract for exclusively owned permutations of [0,size()). + * @details Construction is identity-only; reads are checked and immutable. + * Rotation preserves the bijection and merge rebases the consumed donor. + * Implementations provide identity_impl(n), size_impl(), value_at_impl(i), + * rotate_left_impl(left,right,distance), merge_impl(donor), and + * memory_usage_bytes_impl() with the corresponding facade semantics below. + * size_impl() and memory_usage_bytes_impl() must be noexcept; the other + * extension points propagate the documented exceptions. Validation belongs + * to the implementation, not a second facade check. No virtual dispatch. + * Required signatures are static Impl identity_impl(size_type), + * size_type size_impl() const noexcept, Index value_at_impl(size_type) const, + * void rotate_left_impl(size_type,size_type,size_type), void merge_impl(Impl&), + * and size_type memory_usage_bytes_impl() const noexcept. Private extension + * points grant friendship to this base. Concrete owners default-construct + * canonical empty and are move-only with nonthrowing move and destruction; + * moves leave the source canonical empty and self-move is a no-op. + * @tparam Impl Concrete owning implementation. + * @tparam Index Unsigned integer other than bool with at most 64 value bits. + */ +template +class PermutationBase { + static_assert(std::is_integral_v && std::is_unsigned_v && + !std::is_same_v && + std::numeric_limits::digits <= 64); + + public: + /** @brief Stored index type. @details Independent of the count width. */ + using value_type = Index; + /** @brief Immutable read result. @details Returned by value, never writable. + */ + using const_reference = Index; + /** @brief Count and position type. @details Full unsigned size_t range. */ + using size_type = std::size_t; + + /** + * @brief Construct an owning identity permutation. + * @details identity_impl(n) returns Impl containing 0 through n-1, reclaiming + * all acquired storage on failure. The domain is min(SIZE_MAX,max(Index)+1), + * interpreted mathematically; fixed-width indices never widen. + * @param n Number of entries, represented independently of Index. + * @return Exclusive owner of the identity permutation. + * @throws std::length_error If n exceeds the index domain. + * @throws std::bad_alloc If construction allocation fails. + */ + static Impl identity(size_type n) { return Impl::identity_impl(n); } + /** + * @brief Return the logical index count. + * @details size_impl() returns the count without allocation or mutation. + * @return Number of entries, in [0,SIZE_MAX]. + */ + size_type size() const noexcept { return impl().size_impl(); } + /** + * @brief Test emptiness. + * @details Equivalent to size()==0; canonical empty owns no heap storage. + * @return Whether no entries remain. + */ + bool empty() const noexcept { return size() == 0; } + /** + * @brief Read a checked zero-based index by value. + * @details value_at_impl(position) includes pending biases without pushing + * tags or otherwise modifying storage. No sentinel value is used. + * @param position Position in [0,size()). + * @return Logical index in [0,size()). + * @throws std::out_of_range If position >= size(). + */ + const_reference operator[](size_type position) const { + return impl().value_at_impl(position); + } + /** + * @brief Rotate [left,right) left, preserving outside indices. + * @details rotate_left_impl validates even at zero distance. Empty intervals + * are no-ops; otherwise distance is reduced modulo right-left. Allocation + * failure leaves all contents and pending biases unchanged. + * @param left Inclusive start. + * @param right Exclusive end, at most size(). + * @param distance Left rotation distance in entries. + * @throws std::out_of_range If left > right or right > size(). + * @throws std::bad_alloc If preflight allocation fails. + */ + void rotate_left(size_type left, size_type right, size_type distance) { + impl().rotate_left_impl(left, right, distance); + } + /** + * @brief Append donor indices plus the old receiver size and consume donor. + * @details merge_impl accepts only the same concrete configuration. Self + * merge and empty donor are no-ops; success leaves donor canonical empty. + * Size checks and allocating preflight precede rebasing or ownership edits; + * either failure leaves both participants unchanged. + * @param donor Owner of the permutation to append and rebase. + * @throws std::length_error If the combined count or index domain overflows. + * @throws std::bad_alloc If preflight allocation fails. + */ + void merge(Impl& donor) { impl().merge_impl(donor); } + /** + * @brief Explicitly account for requested live storage. + * @details memory_usage_bytes_impl() traverses allocations in linear time, + * without allocating or throwing. Includes the facade, alignment, metadata, + * and unused payload capacity; saturates at SIZE_MAX. Excludes allocator + * overhead, RSS, stack scratch, and construction/preflight peaks. Concrete + * memory_usage() supplies a structured breakdown, not a second total. + * @return Total requested live bytes including this owner exactly once. + */ + size_type memory_usage_bytes() const noexcept { + return impl().memory_usage_bytes_impl(); + } + + private: + Impl& impl() noexcept { return static_cast(*this); } + const Impl& impl() const noexcept { return static_cast(*this); } +}; +} // namespace pixie diff --git a/include/pixie/permutations/permutation.h b/include/pixie/permutations/permutation.h new file mode 100644 index 0000000..8e0ecef --- /dev/null +++ b/include/pixie/permutations/permutation.h @@ -0,0 +1,246 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Exclusively owned permutation of the integers in [0, size()). + * @details Construction is identity-only. Rotations preserve the bijection; + * consuming merge appends donor entries plus the old receiver size. Fixed-width + * indices never widen. Reads return values, not references. Access, rotation, + * and merge use logarithmic tree work plus bounded local re-encoding; identity + * construction and explicit memory accounting are linear. No public arbitrary + * import, split, element assignment, or value addition is provided. Allocation + * failure leaves existing containers unchanged, including pending lazy index + * biases. Single-leaf and eligible complete-child rotations are nonallocating; + * general rotations use transactional split/join with bounded leaf work. + * Default leaves use 256-byte blocks with 1920 payload bits, wrapped to 320 + * bytes for lazy tags/alignment; F8 tagged nodes use 192 bytes on 64-bit hosts + * versus 128 untagged. Metadata is not promised strictly succinct. There is + * no whole-container contiguous buffer. Internal detail types are unsupported. + * @tparam Index Unsigned integer, excluding bool, with at most 64 value bits. + * @tparam StorageBits Complete packed block budget, excluding tagged wrappers. + * @tparam Fanout Even tree fanout of at least four. + * @tparam Layout Cumulative or individual child measures. + */ +template +class Permutation + : public PermutationBase, + Index> { + friend class PermutationBase; + using block_type = detail::sequence::PackedValueBlock; + using tree_type = + detail::sequence::SequenceTree; + static_assert(std::is_integral_v && std::is_unsigned_v && + !std::is_same_v && + std::numeric_limits::digits <= 64); + + public: + /** @brief Construct empty. @details No allocation or implicit identity data. + */ + Permutation() noexcept = default; + /** @brief Disallow copying. @details Ownership is exclusive. */ + Permutation(const Permutation&) = delete; + /** @brief Disallow copy assignment. @details Ownership is exclusive. */ + Permutation& operator=(const Permutation&) = delete; + /** + * @brief Transfer ownership without allocation. + * @details Source becomes canonical empty; no stored index is rewritten. + * @param other Source whose ownership is consumed. + */ + Permutation(Permutation&& other) noexcept = default; + /** + * @brief Replace ownership without throwing. + * @details Source becomes canonical empty; self-move is a no-op. + * @param other Source whose ownership replaces this owner's storage. + * @return This owner after replacement. + */ + Permutation& operator=(Permutation&& other) noexcept = default; + + /** + * @brief Construct the identity permutation of n entries in a streaming pass. + * @details Uses only bounded field scratch and the tree's construction + * fringe, never a full temporary index vector. All acquired storage is + * reclaimed on failure. The maximum domain is min(SIZE_MAX, max(Index)+1), + * interpreted mathematically without evaluating an overflowing max(Index)+1 + * expression. + * @param n Number of entries, with positions and counts represented by + * size_t. + * @return An owning identity permutation of n entries. + * @throws std::length_error If the chosen index domain is too small. + * @throws std::bad_alloc On construction allocation failure. + */ + private: + static Permutation identity_impl(std::size_t n) { + check_domain(n); + const auto count = + n / block_type::capacity + (n % block_type::capacity != 0); + auto blocks = + std::views::iota(std::size_t{0}, count) | + std::views::transform([n](std::size_t block) { + std::array fields; + const auto start = block * block_type::capacity; + const auto length = std::min(block_type::capacity, n - start); + for (std::size_t i = 0; i < length; ++i) { + fields[i] = static_cast(start + i); + } + return block_type(std::span(fields.data(), length)); + }); + Permutation result; + result.tree_ = tree_type::from_blocks(blocks); + return result; + } + /** @brief Return domain size. @details Exactly the number of stored indices. + * @return Number of logical entries. + */ + std::size_t size_impl() const noexcept { return tree_.size(); } + /** + * @brief Read an immutable logical index by value. + * @details Accumulates lazy biases without normalizing or modifying storage. + * @param position Zero-based position in [0, size()). + * @return The logical index, including all pending biases. + * @throws std::out_of_range If position >= size(). + */ + Index value_at_impl(std::size_t position) const { return tree_[position]; } + /** + * @brief Rotate a half-open interval left, preserving outside entries. + * @details Validates even at zero distance. Empty intervals do nothing; + * otherwise distance is reduced modulo right-left before mutation. + * @param left Inclusive starting position. + * @param right Exclusive ending position, at most size(). + * @param distance Number of entries to rotate left, reduced modulo length. + * @throws std::out_of_range If left > right or right > size(). + * @throws std::bad_alloc On preflight failure; contents remain unchanged. + */ + void rotate_left_impl(std::size_t left, + std::size_t right, + std::size_t distance) { + tree_.rotate_left(left, right, distance); + } + /** + * @brief Append and consume donor, rebasing its indices by the old size(). + * @details Only identical configurations merge. Self-merge and empty donor + * are no-ops. On success donor becomes canonical empty. Checks size/domain + * and allocates all tree spares before attaching a donor bias; every later + * step is nonthrowing. No donor traversal or complete re-encoding is needed. + * @param donor Source whose rebased entries are appended and consumed. + * @throws std::length_error If the combined size exceeds SIZE_MAX or the + * representable index domain. + * @throws std::bad_alloc On preflight failure; both containers stay + * unchanged. + */ + void merge_impl(Permutation& donor) { + if (this == &donor || donor.empty()) { + return; + } + if (donor.size() > std::numeric_limits::max() - this->size()) { + throw std::length_error("Permutation: concatenation size"); + } + check_domain(this->size() + donor.size()); + tree_.merge_rebased(donor.tree_); + } + + public: + /** + * @brief Requested live storage breakdown, excluding allocator bookkeeping. + * @details The five byte categories payload_capacity_bytes, + * block_metadata_bytes, ordering_tree_bytes, tag_padding_bytes, and + * facade_bytes are disjoint and sum to total_bytes. Payload includes unused + * capacity. Tag/padding bytes are incremental over the same untagged layout. + * Counts exclude preflight peaks and RSS; arithmetic saturates at SIZE_MAX. + */ + struct MemoryUsage { + /** @brief Leaf count. @details Counts live leaf allocations only. */ + std::size_t blocks = 0; + /** @brief Internal node count. @details Excludes leaf allocations. */ + std::size_t nodes = 0; + /** @brief Packed payload bytes. @details Includes unused block capacity. */ + std::size_t payload_capacity_bytes = 0; + /** @brief Local bookkeeping bytes. @details Excludes payload and tags. */ + std::size_t block_metadata_bytes = 0; + /** @brief Untagged node bytes. @details Includes their alignment padding. + */ + std::size_t ordering_tree_bytes = 0; + /** @brief Incremental tag bytes. @details Includes additional padding. */ + std::size_t tag_padding_bytes = 0; + /** @brief Inline owner bytes. @details Includes the embedded tree handle. + */ + std::size_t facade_bytes = sizeof(Permutation); + /** @brief Total live bytes. @details Saturated sum of disjoint categories. + */ + std::size_t total_bytes = sizeof(Permutation); + }; + /** + * @brief Explicit linear-time live allocation accounting. + * @details Never called implicitly during queries or mutation; visits each + * allocation once, with no allocation of its own. + * @return Disjoint storage categories and total requested live bytes. + */ + MemoryUsage memory_usage() const noexcept { + using Untagged = detail::sequence::SequenceTree; + const auto tree = tree_.memory_usage(); + MemoryUsage result; + result.blocks = tree.blocks; + result.nodes = tree.nodes; + result.payload_capacity_bytes = tree_type::saturated_multiply( + tree.blocks, block_type{}.payload_capacity_bytes()); + result.block_metadata_bytes = tree_type::saturated_multiply( + tree.blocks, block_type{}.metadata_bytes()); + result.ordering_tree_bytes = + tree_type::saturated_multiply(tree.nodes, Untagged::node_storage_bytes); + result.tag_padding_bytes = tree_type::saturated_add( + tree_type::saturated_multiply( + tree.blocks, tree_type::block_storage_bytes - sizeof(block_type)), + tree_type::saturated_multiply( + tree.nodes, + tree_type::node_storage_bytes - Untagged::node_storage_bytes)); + result.total_bytes = tree_type::saturated_add( + sizeof(*this), + tree_type::saturated_add(tree.block_bytes, tree.node_bytes)); + return result; + } + /** @brief Explicit live footprint query. @details Linear in allocations. + * @return Total requested live bytes, including this facade. + */ + private: + std::size_t memory_usage_bytes_impl() const noexcept { + return memory_usage().total_bytes; + } + +#ifdef PIXIE_SEQUENCE_TREE_TESTING + public: + /** + * @brief Inspect the tree in test builds without exposing mutable ownership. + * @details Provides structural validation and stable identities, not + * rebasing. + * @return Read-only tree reference borrowing this owner's lifetime. + */ + const tree_type& test_tree() const noexcept { return tree_; } +#endif + + private: + static void check_domain(std::size_t n) { + if (n != 0 && n - 1 > std::numeric_limits::max()) { + throw std::length_error("Permutation: index domain"); + } + } + tree_type tree_; +}; + +} // namespace pixie diff --git a/include/pixie/permutations/permutation_implementations.h b/include/pixie/permutations/permutation_implementations.h new file mode 100644 index 0000000..17c4616 --- /dev/null +++ b/include/pixie/permutations/permutation_implementations.h @@ -0,0 +1,58 @@ +#pragma once + +// clang-format off +/** + * @file permutation_implementations.h + * @brief Permutation implementation catalog for benchmarks. + * @details Consumers include pixie/permutations/permutation.h directly. + * + * Current snapshot, 2026-09-21: Ryzen 7 8845HS, WSL/Linux, GCC 13.3, + * -O3 -DNDEBUG -march=native, Google Benchmark 1.9.4, CPU 0, 16 MiB L3. + * uint64_t, B256/F8/cumulative. Ranges span two current-code CPU medians + * (5 repetitions at 0.15s, 9 at 0.2s), not confidence intervals. Host drift + * precludes close rankings or a claim of zero promotion overhead. + * + * | Representation | Read 1 MiB ns | Read 32 MiB ns | Read 128 MiB ns | Owned at 32 MiB | + * | -------------- | ------------- | -------------- | --------------- | --------------- | + * | Perm64 | 46 | 254-269 | 412-466 | 46.32 MiB | + * | Raw64 control | 43-44 | 197-207 | 335-351 | 36.57 MiB | + * + * | Representation | Global 32 MiB us | Local65 32 MiB us | + * | -------------- | ----------------------- | ----------------------- | + * | Perm64 | 37.8-38.4 | 25.8-26.4 | + * | Raw64 control | 32.0-32.5 | 23.7-24.3 | + * + * Useful MiB describes index payload, not total footprint. Reads include hash + * generation, 64 reads/iteration. Rotations use 128 fixed-stream operations, + * 8 batches/repetition; reconstruction/destruction excluded, reconstructed data + * pre-touched. Local65 means elements and can cross leaves, not a hot kernel. + * Equal rebased merge at combined N=4194304: 18.3-19.6 us/merge, two + * independent merges/iteration, 32 iterations; rotated input setup/destruction + * excluded. Short timed mutation totals remain sensitive to host/harness + * variation. + * + * Fresh-tree requested storage includes owner, capacity, metadata, tags, and + * padding, not allocator overhead, RSS, or temporary peaks. Default tagged + * leaf/node allocations are 320/192 bytes versus untagged 256/128; owner is + * 24 bytes. The 9.75 MiB tag/alignment cost at 32 MiB buys lazy donor rebasing. + * Defaults remain conservative, configurable, and not claimed optimal. + * + * Reproduce each filter separately, using the local /bench wrapper: + * @code{.sh} + * flock /tmp/kilo/pixie-experiment-timing.lock \ + * env CPP_BENCH_CPU=0 CPP_JOBS=4 \ + * /home/user/.config/kilo/scripts/cpp-bench native \ + * permutation_benchmarks '' 9 0.2s + * @endcode + * Read filter: + * `^FacadeAccessIndependent/(Perm64|Raw64)_B256_F8_Prefix/N:(131072|4194304|16777216)$` + * Rotation filter: + * `^FacadeRotate(Global|Local65)Batch/(Perm64|Raw64)_B256_F8_Prefix/N:(131072|4194304)/iterations:8$` + * Merge filter: + * `^FacadeMergeEqual/Perm64_B256_F8_Prefix/N:4194304/iterations:32$` + * Fixed-iteration rows ignore the minimum-time argument. Serialize entire + * wrapper invocations; do not overlap builds or tests with timing. + */ +// clang-format on +#include +#include diff --git a/include/pixie/permutations/sequence.h b/include/pixie/permutations/sequence.h new file mode 100644 index 0000000..cb8b26e --- /dev/null +++ b/include/pixie/permutations/sequence.h @@ -0,0 +1,542 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Owning, move-only tree sequence with immutable element reads. + * @details Packed storage returns T by value. Indirect storage returns const T& + * and requires nonthrowing move construction and destruction, but neither + * default construction nor move assignment. Signed integers initially use + * indirect storage. Indirect rotations and concatenating merges never move + * payload objects after construction. References to indirect elements survive + * both, including references obtained from a consumed donor. They expire upon + * destruction of the eventual owner or replacement by move assignment. + * + * Indirect payload lives in independent frozen vectors of actual object slots + * (including actual bool objects). A singly linked owning chain is spliced in + * O(1) after a successful order-tree merge and destroyed iteratively. Repeated + * small merges deliberately retain small chunks and their unused capacity; + * there is no compaction, contiguous-data promise, or reference rebasing. + * Access, rotation and merge use O(Fanout*height) tree work plus bounded local + * block operations. Construction, destruction and explicit accounting may be + * linear. Single-leaf and eligible complete-child rotations do not allocate; + * general rotations use transactional split/join with bounded leaf work. + * Default order blocks occupy 256 bytes with 1920 payload bits; untagged F8 + * nodes occupy 128 bytes on 64-bit hosts. Dynamic metadata is not promised + * strictly succinct. Internal detail types are unsupported. + * No public split, writable access, or serialization is provided. + * @tparam T Unqualified object type; see storage requirements above. + * @tparam Storage Compile-time selection, never a runtime switch. + * @tparam StorageBits Complete local block budget, not whole-tree memory. + * @tparam Fanout Even tree fanout, at least four. + * @tparam Layout Representation of tree child lengths. + * @tparam ChunkBytes Indirect target bytes per vector, at least one T per + * chunk. + */ +template +class PermutableSequence + : public PermutableSequenceBase< + PermutableSequence, + T, + std::conditional_t && + std::is_unsigned_v && + std::numeric_limits::digits <= 64), + T, + const T&>> { + static_assert(std::is_object_v && !std::is_array_v && + std::same_as>); + static constexpr bool packable = std::is_integral_v && + std::is_unsigned_v && + std::numeric_limits::digits <= 64; + static_assert(Storage == ElementStorage::automatic || + Storage == ElementStorage::packed || + Storage == ElementStorage::indirect); + static_assert(Storage != ElementStorage::packed || packable, + "Packed elements must be bool or unsigned integers <=64 bits"); + + public: + /** + * @brief Stored element type. + * @details Identical to T, irrespective of physical representation. + */ + using value_type = T; + /** + * @brief Resolved compile-time storage strategy. + * @details Always packed or indirect; automatic is resolved from T. + */ + static constexpr ElementStorage storage = + Storage == ElementStorage::automatic + ? (packable ? ElementStorage::packed : ElementStorage::indirect) + : Storage; + /** + * @brief Immutable indexed result type. + * @details T by value when packed; const T& with stable indirect ownership. + */ + using const_reference = + std::conditional_t; + + private: + friend class PermutableSequenceBase; + static constexpr bool packed = storage == ElementStorage::packed; + static_assert( + packed || (std::is_nothrow_move_constructible_v && + std::is_nothrow_destructible_v), + "Indirect elements require noexcept move construction/destruction"); + using Block = + std::conditional_t, + detail::sequence::PointerBlock>; + static_assert(sizeof(Block) * CHAR_BIT == StorageBits); + using Tree = detail::sequence::SequenceTree; + static constexpr std::size_t chunk_capacity = + std::max(std::size_t{1}, ChunkBytes / sizeof(T)); + + struct BoolSlot { + bool value; + }; + static_assert(std::is_trivial_v); + using Slot = std::conditional_t, BoolSlot, T>; + static_assert(sizeof(Slot) == sizeof(T)); + struct Chunk { + std::vector values; + Chunk* next = nullptr; + }; + struct Chunks { + Chunk* head = nullptr; + Chunk* tail = nullptr; + Chunks() noexcept = default; + Chunks(const Chunks&) = delete; + Chunks& operator=(const Chunks&) = delete; + Chunks(Chunks&& other) noexcept + : head(std::exchange(other.head, nullptr)), + tail(std::exchange(other.tail, nullptr)) {} + Chunks& operator=(Chunks&& other) noexcept { + if (this != &other) { + clear(); + head = std::exchange(other.head, nullptr); + tail = std::exchange(other.tail, nullptr); + } + return *this; + } + ~Chunks() { clear(); } + void clear() noexcept { + while (head) { + auto* next = head->next; + delete head; + head = next; + } + tail = nullptr; + } + void append(Chunk* chunk) noexcept { + if (tail) { + tail->next = chunk; + } else { + head = chunk; + } + tail = chunk; + } + void splice(Chunks& donor) noexcept { + if (!donor.head) { + return; + } + if (tail) { + tail->next = donor.head; + } else { + head = donor.head; + } + tail = donor.tail; + donor.head = donor.tail = nullptr; + } + }; + struct NoChunks {}; + // Declaration order destroys the pointer tree before the pointed-to objects. + [[no_unique_address]] std::conditional_t chunks_; + Tree tree_; + +#ifdef PIXIE_SEQUENCE_TREE_TESTING + inline static thread_local std::ptrdiff_t payload_failure_ = -1; +#endif + static void payload_allocation() { +#ifdef PIXIE_SEQUENCE_TREE_TESTING + if (payload_failure_ == 0) { + throw std::bad_alloc(); + } + if (payload_failure_ > 0) { + --payload_failure_; + } +#endif + } + static std::size_t add(std::size_t a, std::size_t b) noexcept { + const auto max = std::numeric_limits::max(); + return b > max - a ? max : a + b; + } + static std::size_t multiply(std::size_t a, std::size_t b) noexcept { + const auto max = std::numeric_limits::max(); + return b && a > max / b ? max : a * b; + } + + public: + /** + * @brief Construct canonical empty. + * @details Performs no allocation and owns no payload or tree nodes. + */ + PermutableSequence() noexcept = default; + /** + * @brief Exclusive ownership disallows copying. + * @details This deleted operation cannot duplicate payload ownership. + */ + PermutableSequence(const PermutableSequence&) = delete; + /** + * @brief Exclusive ownership disallows copy assignment. + * @details Transfer ownership with move assignment instead. + */ + PermutableSequence& operator=(const PermutableSequence&) = delete; + /** + * @brief Transfer ownership without moving elements. + * @details The source becomes canonical empty, without allocation. Existing + * indirect references remain valid under the new owner's lifetime. + * @param other Source whose ownership is consumed. + */ + PermutableSequence(PermutableSequence&& other) noexcept = default; + /** + * @brief Replace ownership without moving elements; source becomes empty. + * @details Self-move is a no-op. References to replaced elements expire. + * References to transferred indirect elements remain valid. Reclaims the old + * payload without recursive chunk-chain destruction; performs no allocation. + * @param other Source whose ownership is consumed unless it is this object. + * @return This sequence after ownership replacement. + */ + PermutableSequence& operator=(PermutableSequence&& other) noexcept { + if (this != &other) { + tree_ = std::move(other.tree_); + chunks_ = std::move(other.chunks_); + } + return *this; + } + /** + * @brief Destroy order nodes and iteratively reclaim all owned chunks. + * @details Invalidates references to owned elements; does not throw. + */ + ~PermutableSequence() = default; + + /** + * @brief Consume a possibly single-pass range through ranges::iter_move. + * @details Builds the order tree incrementally, with block-sized stack + * scratch and at most one unpublished chunk; there is no full-input temporary + * or pointer directory. Each chunk reserves before constructing its elements + * and is never resized after publishing addresses. Input need not be sized, + * common, contiguous, default-constructible, or move-assignable. On + * allocation, input, or element-construction failure all acquired ownership + * is reclaimed, but already visited input may have been consumed. No + * caller-owned references survive construction. Exceptions from input + * iteration or element construction propagate without leaking ownership. + * @tparam Range Input range whose iterator-move results can construct T. + * @param range Source consumed in iteration order through ranges::iter_move. + * @return Exclusively owned sequence preserving the input element order. + * @throws std::length_error If the element count exceeds SIZE_MAX or the + * configured chunk capacity exceeds the backing vector's maximum size. + * @throws std::bad_alloc If chunk, vector, or tree allocation fails. + */ + private: + template + requires std:: + constructible_from> + static PermutableSequence from_range_impl(Range&& range) { + PermutableSequence result; + auto it = std::ranges::begin(range); + const auto end = std::ranges::end(range); + Chunk* current = nullptr; + std::size_t offset = 0; + auto has_input = [&] { + if constexpr (packed) { + return it != end; + } else { + return (current && offset < current->values.size()) || it != end; + } + }; + auto next_block = [&] { + std::array scratch; + std::size_t count = 0; + while (count < Block::capacity && has_input()) { + if constexpr (packed) { + scratch[count++] = T(std::ranges::iter_move(it)); + ++it; + } else { + if (!current || offset == current->values.size()) { + payload_allocation(); + auto chunk = std::make_unique(); + payload_allocation(); + chunk->values.reserve(chunk_capacity); + while (chunk->values.size() < chunk_capacity && it != end) { + if constexpr (std::same_as) { + chunk->values.emplace_back(bool(std::ranges::iter_move(it))); + } else { + chunk->values.emplace_back(std::ranges::iter_move(it)); + } + ++it; + } + current = chunk.get(); + result.chunks_.append(chunk.release()); + offset = 0; + } + if constexpr (std::same_as) { + scratch[count++] = std::addressof(current->values[offset++].value); + } else { + scratch[count++] = std::addressof(current->values[offset++]); + } + } + } + return Block( + std::span(scratch.data(), count)); + }; + // A genuine single-pass cursor: dereferencing never advances the input. + // A transform_view would incorrectly promise an equality-preserving map. + struct Blocks { + decltype(next_block)& next; + Block current; + struct Iterator { + using value_type [[maybe_unused]] = Block; + using difference_type [[maybe_unused]] = std::ptrdiff_t; + using iterator_concept [[maybe_unused]] = std::input_iterator_tag; + Blocks* source; + Block& operator*() const noexcept { return source->current; } + Iterator& operator++() { + source->current = source->next(); + return *this; + } + void operator++(int) { ++*this; } + bool operator==(std::default_sentinel_t) const noexcept { + return source->current.size() == 0; + } + }; + Iterator begin() { + current = next(); + return {this}; + } + std::default_sentinel_t end() const noexcept { return {}; } + } blocks{next_block, {}}; + result.tree_ = Tree::from_blocks(blocks); + return result; + } + /** + * @brief Return the logical element count. + * @details Constant time; counts elements, not bytes or bits. + * @return Count in [0,SIZE_MAX]. + */ + std::size_t size_impl() const noexcept { return tree_.size(); } + /** + * @brief Read immutable element i, using zero-based indexing. + * @details Indirect references remain stable through rotation, merge and + * ownership moves, until their eventual owner destroys or replaces them. + * Access uses O(Fanout*height) tree work; no mutable reference is exposed. + * @param i Zero-based element position in [0,size()). + * @return Packed T by value, or a const T& to the owned indirect element. + * @throws std::out_of_range If i >= size(). + */ + const_reference value_at_impl(std::size_t i) const { + if constexpr (packed) { + return tree_[i]; + } else { + return *tree_[i]; + } + } + /** + * @brief Rotate half-open [left,right) left, preserving all outside values. + * @details Validates the range even for distance zero. Reduces distance + * modulo nonempty range length; an empty valid range is a no-op. Allocation + * failure leaves values and indirect addresses unchanged. + * @param left Inclusive start of the half-open range. + * @param right Exclusive end of the half-open range. + * @param distance Left-rotation distance in elements, reduced when nonempty. + * @throws std::out_of_range If left > right or right > size(). + * @throws std::bad_alloc If tree preflight allocation fails. + */ + void rotate_left_impl(std::size_t left, + std::size_t right, + std::size_t distance) { + tree_.rotate_left(left, right, distance); + } + /** + * @brief Append unchanged donor values and consume its exclusive ownership. + * @details Only identical configurations can merge. Self-merge and empty + * donor are no-ops. On success donor is canonical empty. Allocation or size + * failure leaves both participants unchanged. Only after tree commit are + * payload chains spliced in + * O(1), with no chunk traversal, payload move, chunk/vector allocation or + * reallocation. References from either participant retain their addresses. + * @param donor Sequence whose unchanged values follow this sequence's values. + * @throws std::length_error If the combined element count exceeds SIZE_MAX. + * @throws std::bad_alloc If tree preflight allocation fails. + */ + void merge_impl(PermutableSequence& donor) { + if (this == &donor) { + return; + } + tree_.merge(donor.tree_); + if constexpr (!packed) { + chunks_.splice(donor.chunks_); + } + } + + public: + /** + * @brief Explicit live requested-memory breakdown, excluding allocator/RSS. + * @details Total is facade_bytes + tree_block_bytes + tree_node_bytes + + * chunk_header_bytes + vector_capacity_bytes. Other byte/bit fields describe + * subsets and must not be added again. Facade bytes include the embedded tree + * and chain handles; chunk headers include each vector object exactly once. + * Vector capacity includes live slots and slack, with native T alignment. + * Allocations owned internally by T (for example string buffers) are + * excluded. Arithmetic saturates at SIZE_MAX; temporary + * construction/preflight peaks are excluded. This is not an implicit + * mutation-time statistic. + */ + struct MemoryUsage { + /** @brief Inline owner bytes. @details Includes tree and chain handles. */ + std::size_t facade_bytes = sizeof(PermutableSequence); + /** @brief Leaf count. @details Counts live order-tree leaves. */ + std::size_t blocks = 0; + /** @brief Node count. @details Counts internal order-tree allocations. */ + std::size_t nodes = 0; + /** @brief Leaf bytes. @details Includes capacity, metadata, and padding. */ + std::size_t tree_block_bytes = 0; + /** @brief Node bytes. @details Includes internal allocation padding. */ + std::size_t tree_node_bytes = 0; + /** @brief Indirect ordering bytes. @details Subset of tree bytes; zero + * packed. */ + std::size_t order_bytes = 0; + /** @brief Packed payload capacity in bits. @details Subset; zero indirect. + */ + std::size_t packed_capacity_bits = 0; + /** @brief Payload chunk count. @details Zero for packed storage. */ + std::size_t chunks = 0; + /** @brief Chunk header bytes. @details Includes each vector object once. */ + std::size_t chunk_header_bytes = 0; + /** @brief Payload allocation bytes. @details Includes live slots and slack. + */ + std::size_t vector_capacity_bytes = 0; + /** @brief Live payload slot bytes. @details Subset of vector capacity. */ + std::size_t vector_live_bytes = 0; + /** @brief Unused payload slot bytes. @details Subset of vector capacity. */ + std::size_t vector_slack_bytes = 0; + /** @brief Total live bytes. @details Saturated sum of disjoint categories. + */ + std::size_t total_bytes = sizeof(PermutableSequence); + }; + /** + * @brief Enumerate tree allocations and chunks in linear time, without + * writes. + * @details Called only explicitly; merge and rotation never call accounting. + * Performs no allocation and does not throw. Byte arithmetic saturates at + * SIZE_MAX; see MemoryUsage for included and excluded allocation domains. + * @return Requested live-memory breakdown, including this facade exactly + * once. + */ + MemoryUsage memory_usage() const noexcept { + MemoryUsage result; + const auto tree = tree_.memory_usage(); + result.blocks = tree.blocks; + result.nodes = tree.nodes; + result.tree_block_bytes = tree.block_bytes; + result.tree_node_bytes = tree.node_bytes; + const auto tree_bytes = add(tree.block_bytes, tree.node_bytes); + if constexpr (packed) { + result.packed_capacity_bits = + multiply(multiply(tree.blocks, Block::capacity), + std::numeric_limits::digits); + } else { + result.order_bytes = tree_bytes; + for (auto* chunk = chunks_.head; chunk; chunk = chunk->next) { + ++result.chunks; + result.chunk_header_bytes = + add(result.chunk_header_bytes, sizeof(Chunk)); + result.vector_capacity_bytes = + add(result.vector_capacity_bytes, + multiply(chunk->values.capacity(), sizeof(Slot))); + result.vector_live_bytes = + add(result.vector_live_bytes, + multiply(chunk->values.size(), sizeof(Slot))); + result.vector_slack_bytes = + add(result.vector_slack_bytes, + multiply(chunk->values.capacity() - chunk->values.size(), + sizeof(Slot))); + } + } + result.total_bytes = + add(sizeof(*this), add(tree_bytes, add(result.chunk_header_bytes, + result.vector_capacity_bytes))); + return result; + } + /** + * @brief Return live requested bytes, including this object. + * @details Explicit linear-time accounting, equivalent to + * memory_usage().total_bytes; does not allocate or throw. + * @return Requested live bytes, saturated at SIZE_MAX on diagnostic overflow. + */ + private: + std::size_t memory_usage_bytes_impl() const noexcept { + return memory_usage().total_bytes; + } + +#ifdef PIXIE_SEQUENCE_TREE_TESTING + public: + /** + * @brief Test-only order-tree type for existing allocation/structure probes. + * @details All translation units must agree on PIXIE_SEQUENCE_TREE_TESTING. + */ + using test_tree_type = Tree; + /** + * @brief Return a test-only read-only tree view. + * @details No mutable facade escape hatch; the reference borrows this + * object's lifetime and the tree contents may change upon facade mutation. + * @return Const reference to the embedded order tree. + */ + const Tree& test_tree() const noexcept { return tree_; } + /** + * @brief Fail the next payload allocation after n successful allocation + * sites. + * @details -1 disables injection. Sites are chunk new and vector reserve, + * thread-local per facade instantiation; tree allocation has its own hook. + * No global allocation interception or production counter is installed. + * @param n Number of successful allocation sites allowed before bad_alloc; + * pass -1 to disable injection. + */ + static void test_fail_payload_after(std::ptrdiff_t n) noexcept { + payload_failure_ = n; + } +#endif +}; + +} // namespace pixie diff --git a/include/pixie/permutations/sequence_implementations.h b/include/pixie/permutations/sequence_implementations.h new file mode 100644 index 0000000..b3d0ac0 --- /dev/null +++ b/include/pixie/permutations/sequence_implementations.h @@ -0,0 +1,65 @@ +#pragma once + +// clang-format off +/** + * @file sequence_implementations.h + * @brief Permutable-sequence implementation catalog for benchmarks. + * @details Consumers include pixie/permutations/sequence.h directly. + * + * Current snapshot, 2026-09-21: Ryzen 7 8845HS, WSL/Linux, GCC 13.3, + * -O3 -DNDEBUG -march=native, Google Benchmark 1.9.4, CPU 0, 16 MiB L3. + * uint64_t, B256/F8/cumulative, indirect C4096. Ranges span two current-code + * CPU medians (5 repetitions at 0.15s, 9 at 0.2s), not confidence intervals. + * Repeated raw controls also drifted materially; no close ranking or universal + * storage-policy winner is established. + * + * | Representation | Read 1 MiB ns | Read 32 MiB ns | Read 128 MiB ns | Owned at 32 MiB | + * | -------------- | ------------- | -------------- | --------------- | --------------- | + * | Packed64 | 43 | 179-204 | 320-356 | 36.57 MiB | + * | Indirect64 | 41-43 | 169-191 | 326-339 | 68.82 MiB | + * | Raw64 control | 42 | 191-211 | 311-368 | 36.57 MiB | + * + * | Storage | Global 32 MiB us | Local65 32 MiB us | Equal merge 32 MiB us | + * | -------- | ----------------------- | ----------------------- | --------------------- | + * | Packed | 32.3-33.9 | 23.6-23.9 | 12.5-13.1 | + * | Indirect | 32.9-34.0 | 23.6-24.1 | 12.4-13.2 | + * + * Useful MiB describes value payload, not total footprint. Reads include hash + * generation, 64 reads/iteration. Rotation uses 128 fixed-stream operations, + * 8 batches/repetition. Local65 means elements and can cross leaves. Equal + * merge uses combined N=4194304, two independent merges/iteration and 32 + * iterations. Mutation setup/destruction is excluded; rebuilding pre-touches + * storage. Short timed totals remain sensitive to host/harness variation. + * + * Fresh-tree requested bytes include alignment and capacity but exclude + * allocator overhead, RSS, temporary peaks, and allocations inside T. Packed + * and indirect owners are 24 and 40 bytes. Bulk indirect storage at 32 MiB has + * 8192 chunks, 0.25 MiB chunk headers, and no vector slack. Constructing and + * merging 4096 singleton uint64_t sequences retains 4096 chunks and 16.16 MiB, + * including 15.97 MiB vector slack (2.53-2.57 us per singleton build+merge). + * Stable references deliberately retain this slack; bulk construction or an + * explicit smaller ChunkBytes avoids it without implicit compaction. Tests, + * not these timings, establish that merge/rotation do not move payload objects. + * Larger-object read rows inspect only the first word, not complete objects. + * + * Reproduce each filter separately, using the local /bench wrapper: + * @code{.sh} + * flock /tmp/kilo/pixie-experiment-timing.lock \ + * env CPP_BENCH_CPU=0 CPP_JOBS=4 \ + * /home/user/.config/kilo/scripts/cpp-bench native \ + * permutable_sequence_benchmarks '' 9 0.2s + * @endcode + * Read filter: + * `^FacadeAccessIndependent/(Packed64|Indirect64|Raw64)_B256_F8_Prefix(_C4096)?/N:(131072|4194304|16777216)$` + * Rotation filter: + * `^FacadeRotate(Global|Local65)Batch/(Packed64|Indirect64|Raw64)_B256_F8_Prefix(_C4096)?/N:(131072|4194304)/iterations:8$` + * Merge filter: + * `^FacadeMergeEqual/(Packed64|Indirect64)_B256_F8_Prefix(_C4096)?/N:4194304/iterations:32$` + * Singleton filter: + * `^FacadeSingletonBuildMerge/Indirect64_B256_F8_Prefix_C4096/N:4096/iterations:8$` + * Fixed-iteration rows ignore the minimum-time argument. Serialize entire + * wrapper invocations; do not overlap builds or tests with timing. + */ +// clang-format on +#include +#include diff --git a/include/pixie/sequence_options.h b/include/pixie/sequence_options.h new file mode 100644 index 0000000..ff9ae25 --- /dev/null +++ b/include/pixie/sequence_options.h @@ -0,0 +1,15 @@ +#pragma once + +/** + * @file sequence_options.h + * @brief Shared compile-time sequence tuning options. + * @details This header contains no storage engine or concrete implementation. + */ +namespace pixie { +/** + * @brief Representation of tree child lengths. + * @details Cumulative stores prefix ends; individual stores separate lengths. + * Both preserve identical public sequence semantics. + */ +enum class LengthLayout { cumulative, individual }; +} // namespace pixie diff --git a/src/benchmarks/bit_sequence_benchmarks.cpp b/src/benchmarks/bit_sequence_benchmarks.cpp new file mode 100644 index 0000000..86858e0 --- /dev/null +++ b/src/benchmarks/bit_sequence_benchmarks.cpp @@ -0,0 +1,854 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "packed_copy_kernels.h" + +// MiB rows use fixed useful payload sizes; budget_MiB access rows instead bound +// requested live tree storage, excluding allocator bookkeeping and RSS. +// Select exact rows through /bench; do not run the entire size/workload matrix +// for an initial screen. Serialize timing with +// flock /tmp/kilo/pixie-experiment-timing.lock . +// No results from the retired flat-directory implementation apply here. + +namespace { + +using CopyFunction = + void (*)(const uint64_t*, size_t, uint64_t*, size_t, size_t); + +// Fixed addresses and query stream across variants. Layout 0: word-aligned; +// 1: source shifted by 17; 2: mixed bit and cache-line offsets, variable +// lengths. Warm reusable buffers, not a streaming-memory benchmark. Timing +// includes the query lookup, indirect call, boundary work and store barrier, +// not allocation. +void PackedCopy(benchmark::State& state, CopyFunction copy) { + const size_t bytes = state.range(0); + const auto layout = state.range(1); + alignas(64) static std::array input, output; + std::mt19937_64 random(9182); + for (auto& word : input) { + word = random(); + } + struct Query { + size_t source, destination, count; + }; + std::array queries; + for (auto& q : queries) { + q = layout == 0 ? Query{0, 0, bytes * 8} + : layout == 1 + ? Query{17, 0, bytes * 8} + : Query{random() % 512, random() % 512, bytes * 8 - random() % 128}; + } + // Validate controls against a bitwise oracle outside timing, including bits + // outside the copied range. Exhaustive exact-extent checks live in tests. + for (const auto& q : queries) { + output.fill(0xabcdef0123456789ULL); + auto expected = output; + for (size_t i = 0; i < q.count; ++i) { + const auto s = q.source + i; + const auto d = q.destination + i; + const auto mask = uint64_t{1} << (d % 64); + expected[d / 64] = (expected[d / 64] & ~mask) | + (((input[s / 64] >> (s % 64)) & 1) << (d % 64)); + } + copy(input.data(), q.source, output.data(), q.destination, q.count); + if (output != expected) { + state.SkipWithError("Packed copy disagrees with bitwise oracle"); + return; + } + } + size_t i = 0; + int64_t bits = 0; + for (auto _ : state) { + const auto& q = queries[i++ % queries.size()]; + copy(input.data(), q.source, output.data(), q.destination, q.count); + bits += q.count; + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(bits / 8); +} + +const bool packed_copy_registered = [] { + const auto add = [](const char* name, CopyFunction copy) { + auto* benchmark = benchmark::RegisterBenchmark(name, PackedCopy, copy); + benchmark->ArgNames({"bytes", "layout"}); + for (int bytes : {64, 128, 256, 512, 4096, 131072}) { + for (int layout : {0, 1, 2}) { + benchmark->Args({bytes, layout}); + } + } + }; + using namespace packed_copy_benchmark; + add("PackedCopy/production", Production); +#if defined(__AVX512F__) + add("PackedCopy/shift1", Copy); + add("PackedCopy/shift2", Copy); + add("PackedCopy/shift4", Copy); +#if defined(__AVX512VBMI2__) + add("PackedCopy/funnel1", Copy); + add("PackedCopy/funnel2", Copy); + add("PackedCopy/funnel4", Copy); +#endif +#endif + return true; +}(); + +using BitBlock = pixie::detail::sequence::BitBlock<2048>; +using Packed128 = pixie::detail::sequence::PackedBitBlock<128 * 8>; +using Packed256 = pixie::detail::sequence::PackedBitBlock<256 * 8>; +using Packed512 = pixie::detail::sequence::PackedBitBlock<512 * 8>; +using MatchedBitBlock = pixie::detail::sequence::BitBlock; +using DirectBlock = pixie::experimental::PermutedBitBlock; +using MappedBlock = pixie::experimental::PermutedBitBlock; +using pixie::LengthLayout; + +// Keep benchmark traits here rather than requiring introspection aliases in +// the experimental public API. Screen axes, not the full template product. +template +struct Variant { + using block_type = Block; + using tree_type = + pixie::detail::sequence::SequenceTree; + static constexpr auto fanout = Fanout; + static constexpr auto layout = Layout; +}; +using P256F8Raw = Variant; +using P256F8Prefix = Variant; +using P256F4Prefix = Variant; +using P256F4Raw = Variant; +using P256F16Prefix = Variant; +using P256F16Raw = Variant; +using P128F8Prefix = Variant; +using P512F8Prefix = Variant; +using DirectMatchedF8Prefix = + Variant; + +constexpr std::size_t kMiB = 1024 * 1024; +constexpr std::int64_t kQueriesPerIteration = 64; + +// Stateless SplitMix64 finalizer: input words are reproducible by global word +// index, independent of block capacity, without a second sequence-sized buffer. +std::uint64_t Mix(std::uint64_t x) { + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ULL; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebULL; + return x ^ (x >> 31); +} + +template +Block MakeBlock(std::size_t first_bit, std::size_t n) { + std::array words{}; + for (std::size_t i = 0; i < (n + 63) / 64; ++i) { + words[i] = Mix(first_bit / 64 + i + 42); + } + return Block(std::span(words), n); +} + +template +auto MakeTree(std::size_t n, + std::size_t component_bits = V::block_type::capacity) { + using Block = typename V::block_type; + using Tree = typename V::tree_type; + static_assert(Block::capacity % 64 == 0); + const auto count = n / component_bits + (n % component_bits != 0); + // The view yields owning blocks by value; no input/leaf directory is kept. + // All registered component sizes are whole words, even for short components. + auto blocks = + std::views::iota(std::size_t{0}, count) | + std::views::transform([=](std::size_t i) { + const auto start = i * component_bits; + return MakeBlock(start, std::min(component_bits, n - start)); + }); + return Tree::from_blocks(blocks); +} + +std::size_t EnvironmentMiB(const char* name) { + const char* text = std::getenv(name); + if (text == nullptr || *text < '0' || *text > '9') { + return 0; + } + char* end = nullptr; + const auto value = std::strtoull(text, &end, 10); + if (*end != '\0' || value > std::numeric_limits::max() / kMiB) { + return 0; + } + return static_cast(value) * kMiB; +} + +template +bool CheckMemoryBudget(benchmark::State& state, + std::size_t n, + std::size_t extra_bytes = 0) { + // Linux MemAvailable is a preflight hint, not a cgroup limit, reservation or + // measured peak. For a tighter container/process budget set + // PIXIE_SEQUENCE_RAM_MIB. Leave half that available budget as headroom. + std::ifstream input("/proc/meminfo"); + std::string key, rest; + std::size_t kib = 0; + std::size_t available = 0; + while (input >> key >> kib) { + if (key == "MemAvailable:") { + available = kib * 1024; + break; + } + std::getline(input, rest); + } + const auto cap = EnvironmentMiB("PIXIE_SEQUENCE_RAM_MIB"); + if (cap != 0) { + available = available == 0 ? cap : std::min(available, cap); + } + using Block = typename V::block_type; + const auto leaves = n / Block::capacity + (n % Block::capacity != 0); + // Planning estimate only: allow a 256-byte node per leaf and 2x slack for + // repair/construction. Actual requested live storage is reported separately. + const auto estimate = 2 * leaves * (sizeof(Block) + 256) + extra_bytes; + state.counters["available_budget_bytes"] = available; + state.counters["planning_bytes"] = estimate; + if (available == 0 || estimate > available / 2) { + state.SkipWithError( + "Insufficient/unknown available RAM; check " + "PIXIE_SEQUENCE_RAM_MIB before selecting a large row"); + return false; + } + return true; +} + +template +void TreeMemory(benchmark::State& state, const typename V::tree_type& tree) { + using Block = typename V::block_type; + using Tree = typename V::tree_type; + // Final live requested bytes only: no allocator bookkeeping, RSS, allocation + // event counts, or temporary high-water claims. Do not sum overlapping + // fields. node_count() is expected to count internal nodes only; each node + // and each block has one separate allocation under the agreed tree ownership + // model. + const auto memory = tree.memory_usage(); + const auto leaves = memory.blocks; + const auto nodes = memory.nodes; + state.counters["logical_bits"] = tree.size(); + state.counters["useful_bytes"] = static_cast(tree.size()) / 8; + state.counters["payload_capacity_bytes"] = + leaves * ((Block::capacity + 7) / 8); + state.counters["internal_bytes"] = memory.node_bytes; + state.counters["owned_bytes"] = memory.total_bytes; + state.counters["bytes_per_bit"] = + static_cast(memory.total_bytes) / tree.size(); + state.counters["leaf_object_bytes"] = leaves * sizeof(Block); + state.counters["leaf_allocation_bytes"] = memory.block_bytes; + state.counters["tree_object_bytes"] = sizeof(tree); + state.counters["blocks"] = leaves; + state.counters["internal_nodes"] = nodes; + state.counters["live_allocations"] = leaves + nodes; + state.counters["height"] = tree.height(); + state.counters["block_capacity_bits"] = Block::capacity; + state.counters["sizeof_block"] = sizeof(Block); + state.counters["sizeof_leaf_allocation"] = Tree::block_storage_bytes; + state.counters["sizeof_node_allocation"] = Tree::node_storage_bytes; + state.counters["alignof_block"] = alignof(Block); + state.counters["leaf_occupancy"] = + static_cast(tree.size()) / (leaves * Block::capacity); + state.counters["fanout"] = V::fanout; + state.counters["cumulative_lengths"] = V::layout == LengthLayout::cumulative; + const auto l3 = EnvironmentMiB("PIXIE_SEQUENCE_L3_MIB"); + if (l3 != 0) { + state.counters["declared_l3_bytes"] = l3; + state.counters["internal_over_l3"] = + static_cast(memory.node_bytes) / l3; + } +} + +template +auto MakeBudgetTree(std::size_t budget) { + using Block = typename V::block_type; + // Start above the target: leaf objects alone use almost the entire budget. + // Trim an owning suffix, at block boundaries, to include actual internal-node + // storage in the budget. No simultaneous replacement tree is constructed. + auto tree = MakeTree((budget / sizeof(Block)) * Block::capacity); + for (auto owned = tree.memory_usage_bytes(); owned > budget; + owned = tree.memory_usage_bytes()) { + const auto leaves = tree.size() / Block::capacity; + const auto retained = std::min(leaves - 1, leaves * budget / owned); + auto suffix = tree.split_off(retained * Block::capacity); + } + return tree; +} + +template +void TreeAccess(benchmark::State& state) { + const std::size_t bytes = state.range(0) * kMiB; + const auto initial_bits = + MemoryBudget + ? (bytes / sizeof(typename V::block_type)) * V::block_type::capacity + : bytes * 8; + if (!CheckMemoryBudget(state, initial_bits)) { + return; + } + const auto l3 = EnvironmentMiB("PIXIE_SEQUENCE_L3_MIB"); + if constexpr (RequireColdNavigation) { + if (l3 == 0) { + state.SkipWithError( + "Declare measured shared L3 via PIXIE_SEQUENCE_L3_MIB"); + return; + } + } + auto tree = [&] { + if constexpr (MemoryBudget) { + return MakeBudgetTree(bytes); + } else { + return MakeTree(initial_bits); + } + }(); + const auto n = tree.size(); + TreeMemory(state, tree); + if constexpr (MemoryBudget) { + state.counters["requested_budget_bytes"] = bytes; + state.counters["budget_utilization"] = + static_cast(tree.memory_usage_bytes()) / bytes; + if (tree.memory_usage_bytes() < bytes * 99 / 100) { + state.SkipWithError("Requested-memory comparison undershot by over 1%"); + return; + } + } + if constexpr (RequireColdNavigation) { + if (tree.internal_memory_bytes() <= l3) { + state.SkipWithError( + "Internal navigation storage does not exceed declared L3"); + return; + } + } + // Construction has written/touched every live allocation, outside timing. + // A full-width counter supplies fresh positions over all N, not a query pool. + // Both streams include hash/modulo generation. The dependent stream feeds + // the returned random input bit into the next hash, not just a fixed branch. + std::uint64_t counter = 123; + std::uint64_t previous = 0; + state.SetLabel(Dependent + ? "64 dependent reads/iteration; generator included" + : "64 independent reads/iteration; generator included"); + for (auto _ : state) { + for (std::int64_t i = 0; i < kQueriesPerIteration; ++i) { + auto key = counter++; + if constexpr (Dependent) { + key ^= previous * 0x9e3779b97f4a7c15ULL; + } + auto bit = tree[Mix(key) % n]; + benchmark::DoNotOptimize(bit); + if constexpr (Dependent) { + previous = static_cast(bit); + } + } + } + state.counters["query_state_bytes"] = + sizeof(counter) + (Dependent ? sizeof(previous) : 0); + state.counters["queries_per_iteration"] = kQueriesPerIteration; + state.SetItemsProcessed(state.iterations() * kQueriesPerIteration); +} + +enum class Mutation { + SplitRejoinRandom, + SplitRejoinBoundary, + Local, + Whole, + Aligned, + Unaligned +}; + +template +void TreeMutation(benchmark::State& state) { + using Block = typename V::block_type; + const std::size_t n = OneLeaf ? state.range(0) : state.range(0) * kMiB * 8; + if (!CheckMemoryBudget(state, n)) { + return; + } + auto tree = MakeTree(n); + std::uint64_t counter = 128; + state.SetLabel( + "steady state; generator+allocation+repair included; no reset"); + for (auto _ : state) { + const auto random = Mix(counter++); + if constexpr (Operation == Mutation::SplitRejoinRandom || + Operation == Mutation::SplitRejoinBoundary) { + // Grid boundaries match the initial build; repair can change leaf seams. + const auto cut = + Operation == Mutation::SplitRejoinRandom + ? random % (n + 1) + : (random % (n / Block::capacity + 1)) * Block::capacity; + auto suffix = tree.split_off(cut); + tree.merge(suffix); + } else if constexpr (Operation == Mutation::Local) { + const auto left = random % (n - 65 + 1); + tree.rotate_left(left, left + 65, 1); + } else if constexpr (Operation == Mutation::Whole) { + tree.rotate_left(0, n, 1 + random % (n - 1)); + } else if constexpr (Operation == Mutation::Aligned) { + // Fanout*capacity grid and distance align initial bottom-level subtrees. + // Subsequent repair need not preserve that geometry; no reset is hidden. + constexpr auto unit = V::fanout * Block::capacity; + const auto units = n / unit; + const auto length = units / 2; + const auto left = (random % (units - length + 1)) * unit; + tree.rotate_left(left, left + length * unit, + (1 + random % (length - 1)) * unit); + } else { + // Large, changing, generally unaligned intervals cover the full sequence. + const auto length = n / 2 + random % (n / 4); + const auto left = Mix(random) % (n - length + 1); + tree.rotate_left(left, left + length, 1 + Mix(random + 1) % (length - 1)); + } + benchmark::DoNotOptimize(tree); + benchmark::ClobberMemory(); + } + if (tree.size() != n) { + state.SkipWithError("Mutation changed the logical size"); + return; + } + TreeMemory(state, tree); + state.counters["max_external_roots"] = + Operation == Mutation::SplitRejoinRandom || + Operation == Mutation::SplitRejoinBoundary + ? 2 + : 1; + state.counters["query_state_bytes"] = sizeof(counter); + state.SetItemsProcessed(state.iterations()); +} + +/** + * @brief Rotate exact child boundaries in freshly built, identical trees. + * @details Each timed batch mutates 64 independent full F^3-leaf trees once. + * Level 3 covers the root; levels 1 and 2 cover its first descendant chain. + * Full selects all children; partial selects children [1,F-1). Construction + * and destruction are excluded; no evolving geometry or testing hooks occur. + */ +void TreeRotateChildBoundary(benchmark::State& state) { + using V = P256F8Prefix; + constexpr auto f = V::fanout; + constexpr auto capacity = V::block_type::capacity; + constexpr auto n = f * f * f * capacity; + constexpr std::size_t batch = 64; + std::array trees; + std::size_t unit = capacity; + for (std::int64_t level = 1; level < state.range(0); ++level) { + unit *= f; + } + const auto left = state.range(1) ? 0 : unit; + const auto right = state.range(1) ? f * unit : (f - 1) * unit; + state.SetLabel("64 identical trees; one rotation/tree; rebuild excluded"); + for (auto _ : state) { + state.PauseTiming(); + for (auto& tree : trees) { + tree = {}; + tree = MakeTree(n); + } + state.ResumeTiming(); + for (auto& tree : trees) { + tree.rotate_left(left, right, 2 * unit); + benchmark::DoNotOptimize(tree); + } + benchmark::ClobberMemory(); + } + state.counters["operations_per_iteration"] = batch; + state.SetItemsProcessed(state.iterations() * batch); +} + +template +void TreeMergeResetSplit(benchmark::State& state) { + const std::size_t n = state.range(0) * kMiB * 8; + if (!CheckMemoryBudget(state, n)) { + return; + } + auto tree = MakeTree(n); + const auto cut = Equal ? n / 2 : n - n / 64; + state.SetLabel( + "merge only; reset split excluded, warms paths; final footprint"); + state.counters["donor_bits"] = n - cut; + // Record actual input heights, not an assumption based on logical lengths. + auto donor = tree.split_off(cut); + state.counters["initial_receiver_height"] = tree.height(); + state.counters["initial_donor_height"] = donor.height(); + tree.merge(donor); + for (auto _ : state) { + state.PauseTiming(); + donor = tree.split_off(cut); + state.ResumeTiming(); + tree.merge(donor); + benchmark::DoNotOptimize(tree); + benchmark::ClobberMemory(); + } + if (tree.size() != n || !donor.empty()) { + state.SkipWithError("Merge failed size/consuming-donor invariant"); + return; + } + TreeMemory(state, tree); + state.counters["max_external_roots"] = 2; + state.SetItemsProcessed(state.iterations()); +} + +template +void TreeSplitResetMerge(benchmark::State& state) { + const std::size_t n = state.range(0) * kMiB * 8; + if (!CheckMemoryBudget(state, n)) { + return; + } + auto tree = MakeTree(n); + std::uint64_t counter = 128; + state.SetLabel("split+generator; reset merge excluded, warms paths"); + for (auto _ : state) { + auto suffix = tree.split_off(Mix(counter++) % (n + 1)); + benchmark::DoNotOptimize(tree); + benchmark::DoNotOptimize(suffix); + state.PauseTiming(); + tree.merge(suffix); + state.ResumeTiming(); + } + TreeMemory(state, tree); + state.SetItemsProcessed(state.iterations()); +} + +template +void TreeMergeSmallComponents(benchmark::State& state) { + using Block = typename V::block_type; + using Tree = typename V::tree_type; + const std::size_t n = state.range(0) * 1024 * 8; + if (!CheckMemoryBudget(state, n)) { + return; + } + state.SetLabel( + "64-bit component generation+construction+consuming merges; " + "final destruction excluded"); + bool reported = false; + for (auto _ : state) { + Tree tree; + for (std::size_t first = 0; first < n; first += 64) { + std::array input{MakeBlock(first, 64)}; + auto component = Tree::from_blocks(input); + tree.merge(component); + } + benchmark::DoNotOptimize(tree); + benchmark::ClobberMemory(); + state.PauseTiming(); + if (!reported) { + TreeMemory(state, tree); + reported = true; + } + tree = Tree{}; + state.ResumeTiming(); + } + state.counters["components_per_iteration"] = n / 64; + state.SetItemsProcessed(state.iterations() * (n / 64)); + state.SetBytesProcessed(state.iterations() * (n / 8)); +} + +template +void TreeBuild(benchmark::State& state) { + using Tree = typename V::tree_type; + const std::size_t n = state.range(0) * kMiB * 8; + if (!CheckMemoryBudget(state, n)) { + return; + } + const auto component_bits = SmallComponents ? 64 : V::block_type::capacity; + state.SetLabel("streaming input generation+build; destruction excluded"); + state.counters["input_component_bits"] = component_bits; + state.counters["input_components"] = + n / component_bits + (n % component_bits != 0); + bool reported = false; + for (auto _ : state) { + auto tree = MakeTree(n, component_bits); + benchmark::DoNotOptimize(tree); + benchmark::ClobberMemory(); + state.PauseTiming(); + if (!reported) { + TreeMemory(state, tree); + reported = true; + } + tree = Tree{}; + state.ResumeTiming(); + } + state.SetItemsProcessed(state.iterations() * n); + state.SetBytesProcessed(state.iterations() * (n / 8)); +} + +template +void TreeMaterializeTraversal(benchmark::State& state) { + using Block = typename V::block_type; + const std::size_t n = state.range(0) * kMiB * 8; + const auto words = n / 64 + (n % 64 != 0); + if (!CheckMemoryBudget(state, n, words * sizeof(std::uint64_t))) { + return; + } + const auto tree = MakeTree(n); + // This is a second full output buffer, not in-place finalization. Allocate, + // initialize and pre-touch it outside timing; every valid bit is overwritten + // during each traversal. No transient allocation/high-water claim is made. + std::vector output(words, 0); + benchmark::DoNotOptimize(output.data()); + benchmark::ClobberMemory(); + state.SetLabel( + "ordered traversal+block flatten+copy; second output buffer; " + "allocation+initialization+destruction excluded"); + std::size_t offset = 0; + for (auto _ : state) { + offset = 0; + tree.for_each_block([&](const Block& block) { + const auto flat = block.flatten(); + pixie::copy_packed_bits(flat.data(), 0, output.data(), offset, + block.size()); + offset += block.size(); + }); + // Never copy a block's physical padding. Clear final output padding even + // for a future non-word-sized case, without a redundant full-buffer clear. + if (n % 64 != 0) { + output.back() &= (std::uint64_t{1} << (n % 64)) - 1; + } + benchmark::DoNotOptimize(output.data()); + benchmark::ClobberMemory(); + } + if (offset != n) { + state.SkipWithError("Block traversal did not materialize the logical size"); + return; + } + for (std::size_t i = 0; i < words; ++i) { + auto expected = Mix(i + 42); + if (i + 1 == words && n % 64 != 0) { + expected &= (std::uint64_t{1} << (n % 64)) - 1; + } + if (output[i] != expected) { + state.SkipWithError("Materialized contents differ from generated input"); + return; + } + } + TreeMemory(state, tree); + const auto output_bytes = output.capacity() * sizeof(std::uint64_t); + state.counters["output_useful_bytes"] = (n + 7) / 8; + state.counters["output_capacity_bytes"] = output_bytes; + state.counters["live_output_allocations"] = 1; + state.counters["tree_plus_output_bytes"] = + tree.memory_usage_bytes() + output_bytes; + state.SetItemsProcessed(state.iterations() * n); + state.SetBytesProcessed(state.iterations() * ((n + 7) / 8)); +} + +void TreeSizes(benchmark::internal::Benchmark* benchmark) { + benchmark->ArgName("MiB"); + for (const auto mib : {1, 8, 32, 128, 512}) { + benchmark->Arg(mib); + } +} + +template +void RegisterTreeVariant(const char* variant) { + const auto add = [=](const char* operation, auto function) { + const auto name = std::string(operation) + "/" + variant; + benchmark::RegisterBenchmark(name.c_str(), function)->Apply(TreeSizes); + }; + add("TreeAccessIndependent", TreeAccess); + add("TreeAccessDependent", TreeAccess); + const auto budget_name = + std::string("TreeAccessBudgetIndependent/") + variant; + auto* budget = benchmark::RegisterBenchmark( + budget_name.c_str(), TreeAccess); + budget->ArgName("budget_MiB"); + for (const auto mib : {4, 16, 64, 256}) { + budget->Arg(mib); + } + add("TreeSplitResetMerge", TreeSplitResetMerge); + add("TreeSplitRejoinRandom", TreeMutation); + add("TreeSplitRejoinCapacityGrid", + TreeMutation); + add("TreeRotateLocal65", TreeMutation); + add("TreeRotateWhole", TreeMutation); + add("TreeRotateGlobalSubtreeGrid", TreeMutation); + add("TreeRotateGlobalUnaligned", TreeMutation); + add("TreeMergeEqualResetSplit", TreeMergeResetSplit); + add("TreeMergeUnequalResetSplit", TreeMergeResetSplit); + add("TreeBuild", TreeBuild); +} + +const bool tree_registered = [] { + benchmark::RegisterBenchmark( + "TreeRotateLocal65/P256F8Prefix", + TreeMutation) + ->ArgName("Bits") + ->Arg(65) + ->Arg(128) + ->Arg(P256F8Prefix::block_type::capacity); + benchmark::RegisterBenchmark("TreeRotateChildBoundary/P256F8Prefix", + TreeRotateChildBoundary) + ->ArgNames({"Level", "Full"}) + ->Args({1, 0}) + ->Args({1, 1}) + ->Args({2, 0}) + ->Args({2, 1}) + ->Args({3, 0}) + ->Args({3, 1}) + ->Iterations(8); + RegisterTreeVariant("P256F8Raw"); + RegisterTreeVariant("P256F8Prefix"); + RegisterTreeVariant("P256F4Prefix"); + RegisterTreeVariant("P256F4Raw"); + RegisterTreeVariant("P256F16Prefix"); + RegisterTreeVariant("P256F16Raw"); + RegisterTreeVariant("P128F8Prefix"); + RegisterTreeVariant("P512F8Prefix"); + RegisterTreeVariant("DirectMatchedF8Prefix"); + // These expensive/conditional probes are baseline-only for the first screen. + benchmark::RegisterBenchmark("TreeBuildSmallComponents/P256F8Prefix", + TreeBuild) + ->Apply(TreeSizes); + benchmark::RegisterBenchmark("TreeMergeSmallComponents/P256F8Prefix", + TreeMergeSmallComponents) + ->ArgName("KiB") + ->Arg(4) + ->Arg(64) + ->Arg(1024); + benchmark::RegisterBenchmark("TreeMaterializeTraversal/P256F8Prefix", + TreeMaterializeTraversal) + ->Apply(TreeSizes); + benchmark::RegisterBenchmark("TreeNavigationBeyondL3/P256F8Prefix", + TreeAccess) + ->ArgName("MiB") + ->Arg(512); + return true; +}(); + +template +void BlockMemory(benchmark::State& state, std::size_t n) { + state.counters["logical_bits"] = n; + state.counters["block_capacity_bits"] = Block::capacity; + state.counters["sizeof_block"] = sizeof(Block); + state.counters["alignof_block"] = alignof(Block); + state.counters["payload_capacity_bytes"] = (Block::capacity + 7) / 8; + state.counters["live_heap_allocations"] = 0; +} + +struct BlockRotation { + std::size_t left, right, distance; +}; + +// 256 precomputed operations, seed 128. Modes: aligned proper subranges, +// unaligned proper subranges, alternating 50/50, whole block, wrapped partial. +// Random generation and construction excluded; repeated mutation is timed. +// N is capacity (or capacity-3), not a fixed logical size across all types. +// Packed256 and MatchedBitBlock match capacity; the other controls need not. +template +void BlockRotate(benchmark::State& state) { + const auto mode = state.range(0); + constexpr std::size_t capacity = Block::capacity; + const std::size_t n = mode == 4 ? capacity - 3 : capacity; + auto block = MakeBlock(0, n); + std::mt19937_64 random(128); + std::array queries; + for (std::size_t i = 0; i < queries.size(); ++i) { + auto& q = queries[i]; + const auto l = random() % 8; + const auto r = l + 2 + random() % (capacity / 128 - 2 - l); + q = {l * 128, r * 128, 128 * (1 + random() % (r - l - 1))}; + if (mode == 1 || mode == 4 || (mode == 2 && i % 2)) { + q = {q.left + 1, q.right - 3, q.distance + 1}; + } + if (mode == 3) { + q = {0, n, 65}; + } + } + if (mode == 4) { + block.rotate_left(0, n, n - 7); + } + std::size_t i = 0; + for (auto _ : state) { + const auto& q = queries[i++ % queries.size()]; + block.rotate_left(q.left, q.right, q.distance); + benchmark::DoNotOptimize(block); + benchmark::ClobberMemory(); + } + BlockMemory(state, n); + state.counters["query_pool_bytes"] = sizeof(queries); + state.SetItemsProcessed(state.iterations()); +} + +template +void BlockRead(benchmark::State& state) { + const std::size_t n = Block::capacity - state.range(0); + auto block = MakeBlock(0, n); + const auto aligned_end = (Block::capacity / 128 - 1) * 128; + block.rotate_left(128, aligned_end, 384); + block.rotate_left(256, aligned_end - 128, 640); + block.rotate_left(0, n, 65); + // Keep the map/origin as runtime state rather than constant-folding setup. + benchmark::DoNotOptimize(block); + std::array queries; + if constexpr (!Flatten) { + std::mt19937_64 random(123); + for (auto& q : queries) { + q = random() % n; + } + } + std::size_t i = 0; + for (auto _ : state) { + if constexpr (Flatten) { + auto flat = block.flatten(); + benchmark::DoNotOptimize(flat); + benchmark::ClobberMemory(); + } else { + benchmark::DoNotOptimize(block[queries[i++ % queries.size()]]); + } + } + BlockMemory(state, n); + state.counters["query_pool_bytes"] = Flatten ? 0 : sizeof(queries); + state.SetItemsProcessed(state.iterations() * (Flatten ? n : 1)); + if constexpr (Flatten) { + state.SetBytesProcessed(state.iterations() * ((n + 7) / 8)); + } +} + +BENCHMARK_TEMPLATE(BlockRotate, DirectBlock)->DenseRange(0, 4); +BENCHMARK_TEMPLATE(BlockRotate, MappedBlock)->DenseRange(0, 4); +BENCHMARK_TEMPLATE(BlockRotate, BitBlock)->DenseRange(0, 4); +BENCHMARK_TEMPLATE(BlockRotate, Packed256)->DenseRange(0, 4); +BENCHMARK_TEMPLATE(BlockRotate, MatchedBitBlock)->DenseRange(0, 4); +BENCHMARK_TEMPLATE(BlockRead, DirectBlock, false) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); +BENCHMARK_TEMPLATE(BlockRead, MappedBlock, false) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); +BENCHMARK_TEMPLATE(BlockRead, BitBlock, false) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); +BENCHMARK_TEMPLATE(BlockRead, Packed256, false) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); +BENCHMARK_TEMPLATE(BlockRead, MatchedBitBlock, false) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); +BENCHMARK_TEMPLATE(BlockRead, DirectBlock, true) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); +BENCHMARK_TEMPLATE(BlockRead, MappedBlock, true) + ->ArgName("unused_bits") + ->Arg(0) + ->Arg(3); + +} // namespace diff --git a/src/benchmarks/packed_copy_kernels.h b/src/benchmarks/packed_copy_kernels.h new file mode 100644 index 0000000..f4fe677 --- /dev/null +++ b/src/benchmarks/packed_copy_kernels.h @@ -0,0 +1,132 @@ +#ifndef PIXIE_BENCHMARKS_PACKED_COPY_KERNELS_H_ +#define PIXIE_BENCHMARKS_PACKED_COPY_KERNELS_H_ + +#include + +// Unrolling experiment, 2026-09-16: Ryzen 7 8845HS, Linux/WSL, GCC 13.3, +// -O3 -march=native, CPU 0. Median CPU ns, five randomized 0.2s repetitions; +// confirmation run after the full aligned/shifted/mixed sweep. Warm buffers, +// runtime indirect calls, setup excluded. Layout 1: source bit 17, destination +// bit 0; layout 2: mixed offsets and lengths. Registered in +// bit_sequence_benchmarks: cpp-bench native bit_sequence_benchmarks +// '^PackedCopy/(shift1|funnel[124])/bytes:[0-9]+/layout:[12]$' 5 0.2s +// +// | bytes | layout | shift1 ns | funnel1 ns | funnel2 ns | funnel4 ns | +// | ----: | -----: | --------: | ---------: | ---------: | ---------: | +// | 128 | 1 | 4.8 | 4.7 | 5.5 | 6.1 | +// | 256 | 1 | 6.8 | 7.9 | 8.8 | 7.9 | +// | 256 | 2 | 11.0 | 10.4 | 10.6 | 10.6 | +// | 4096 | 2 | 68.0 | 63.3 | 66.7 | 64.5 | +// +// No consistent unrolling gain across both runs; production keeps rolled +// funnel. Controls retained for Intel testing, not as public library +// implementations. nm/objdump: funnel 1x/2x/4x = 1011/1085/1282 bytes, 1/2/4 +// VPSHRDVQ per main iteration (7/13/21 total instructions). GCC also unrolls +// the bounded remainder; the rolled main loop is NOT automatically unrolled. No +// hardware counters taken. + +namespace packed_copy_benchmark { + +// Benchmark-only controls for Intel/AMD comparisons. Same contract and boundary +// work as copy_packed_bits; only the shifted AVX-512 loop varies. Keep out of +// the public API. Noinline gives every variant the same runtime-call boundary +// and leaves inspectable symbols for checking actual unrolling and code size. +#if defined(__AVX512F__) +template +__attribute__((noinline)) void Copy(const uint64_t* source, + size_t source_bit, + uint64_t* destination, + size_t destination_bit, + size_t count) { + static_assert(Unroll == 1 || Unroll == 2 || Unroll == 4); + const auto boundary = [&](size_t width) { + const auto shift = source_bit % 64; + uint64_t value = source[source_bit / 64] >> shift; + if (width > 64 - shift) { + value |= source[source_bit / 64 + 1] << (64 - shift); + } + const auto offset = destination_bit % 64; + const auto mask = first_bits_mask(width) << offset; + auto& word = destination[destination_bit / 64]; + word = (word & ~mask) | ((value << offset) & mask); + source_bit += width; + destination_bit += width; + count -= width; + }; + if (count == 0) { + return; + } + if (destination_bit % 64 != 0) { + boundary(std::min(count, 64 - destination_bit % 64)); + } + const auto shift = source_bit % 64; + const auto* input = source + source_bit / 64; + auto* output = destination + destination_bit / 64; + auto words = count / 64; + if (shift == 0) { + std::copy_n(input, words, output); + } else { + const auto low_shift = _mm_cvtsi64_si128(shift); + const auto high_shift = _mm_cvtsi64_si128(64 - shift); +#if defined(__AVX512VBMI2__) + const auto shifts = _mm512_set1_epi64(shift); +#else + static_assert(!Funnel, "Funnel control requires AVX512VBMI2"); +#endif + const auto vector = [&](size_t offset) { + const auto low = _mm512_loadu_si512(input + offset); + const auto high = _mm512_loadu_si512(input + offset + 1); +#if defined(__AVX512VBMI2__) + if constexpr (Funnel) { + _mm512_storeu_si512(output + offset, + _mm512_shrdv_epi64(low, high, shifts)); + } else +#endif + { + _mm512_storeu_si512( + output + offset, + _mm512_or_si512(_mm512_srl_epi64(low, low_shift), + _mm512_sll_epi64(high, high_shift))); + } + }; + for (; words >= 8 * Unroll; + words -= 8 * Unroll, input += 8 * Unroll, output += 8 * Unroll) { + vector(0); + if constexpr (Unroll >= 2) { + vector(8); + } + if constexpr (Unroll >= 4) { + vector(16); + vector(24); + } + } + if constexpr (Unroll != 1) { + for (; words >= 8; words -= 8, input += 8, output += 8) { + vector(0); + } + } + for (size_t i = 0; i < words; ++i) { + output[i] = (input[i] >> shift) | (input[i + 1] << (64 - shift)); + } + } + source_bit += count / 64 * 64; + destination_bit += count / 64 * 64; + count %= 64; + if (count != 0) { + boundary(count); + } +} +#endif + +__attribute__((noinline)) inline void Production(const uint64_t* source, + size_t source_bit, + uint64_t* destination, + size_t destination_bit, + size_t count) { + pixie::copy_packed_bits(source, source_bit, destination, destination_bit, + count); +} + +} // namespace packed_copy_benchmark + +#endif // PIXIE_BENCHMARKS_PACKED_COPY_KERNELS_H_ diff --git a/src/benchmarks/permutable_sequence_benchmarks.cpp b/src/benchmarks/permutable_sequence_benchmarks.cpp new file mode 100644 index 0000000..c12e475 --- /dev/null +++ b/src/benchmarks/permutable_sequence_benchmarks.cpp @@ -0,0 +1,56 @@ +#include + +#include "sequence_facade_benchmarks.h" + +namespace { +using namespace pixie; +template +struct ElementVariant : Configuration { + using sequence_type = + PermutableSequence; + static constexpr bool rebased = false; + static constexpr bool indirect = + Storage == ElementStorage::indirect || !std::is_unsigned_v; + static constexpr std::size_t chunk_bytes = indirect ? ChunkBytes : 0; + static auto Make(std::size_t n) { + // Prvalue elements include move-only values. No full input/pointer vector. + auto input = + std::views::iota(std::size_t{0}, n) | + std::views::transform([](std::size_t i) { return Value(i); }); + return sequence_type::from_range(input); + } +}; + +const bool registered = [] { + Register, true>( + "Packed64_B256_F8_Prefix"); + Register, true>( + "Indirect64_B256_F8_Prefix_C4096"); + Register>( + "PackedBool_B256_F8_Prefix"); + Register, true>( + "IndirectBool_B256_F8_Prefix_C4096"); + Register>>("Payload8_B256_F8_Prefix_C4096"); + Register>, true>("Payload64_B256_F8_Prefix_C4096"); + Register>>("Payload256_B256_F8_Prefix_C4096"); + Register, true>( + "MoveOnly64_B256_F8_Prefix_C4096"); + + Register>("Raw16_B256_F8_Prefix"); + Register>("Raw32_B256_F8_Prefix"); + Register, true>("Raw64_B256_F8_Prefix"); + Register, + false, true>("Indirect64_B256_F8_Prefix_C1024"); + Register, + false, true>("Indirect64_B256_F8_Prefix_C16384"); + + return true; +}(); +} // namespace diff --git a/src/benchmarks/permutation_benchmarks.cpp b/src/benchmarks/permutation_benchmarks.cpp new file mode 100644 index 0000000..2eb380e --- /dev/null +++ b/src/benchmarks/permutation_benchmarks.cpp @@ -0,0 +1,44 @@ +#include + +#include "sequence_facade_benchmarks.h" + +namespace { +using namespace pixie; +template +struct PermutationVariant : Configuration { + using sequence_type = Permutation; + static constexpr bool rebased = true; + static constexpr bool indirect = false; + static constexpr std::size_t chunk_bytes = 0; + static auto Make(std::size_t n) { return sequence_type::identity(n); } +}; + +const bool registered = [] { + Register, true>("Perm16_B256_F8_Prefix"); + Register>("Perm32_B256_F8_Prefix"); + Register, true>("Perm64_B256_F8_Prefix"); + Register>("Raw16_B256_F8_Prefix"); + Register>("Raw32_B256_F8_Prefix"); + Register, true>("Raw64_B256_F8_Prefix"); + // One-axis controls; pair with Perm64_B256_F8_Prefix at the same N. + Register, false, true>( + "Perm64_B256_F4_Prefix"); + Register, false, true>( + "Perm64_B256_F16_Prefix"); + Register, + false, true>("Perm64_B256_F8_Raw"); + Register, + false, true>("Perm64_B256_F4_Raw"); + Register< + PermutationVariant, + false, true>("Perm64_B256_F16_Raw"); + Register, false, true>( + "Perm64_B128_F8_Prefix"); + Register, false, true>( + "Perm64_B512_F8_Prefix"); + return true; +}(); +} // namespace diff --git a/src/benchmarks/sequence_controls_benchmarks.cpp b/src/benchmarks/sequence_controls_benchmarks.cpp new file mode 100644 index 0000000..931648c --- /dev/null +++ b/src/benchmarks/sequence_controls_benchmarks.cpp @@ -0,0 +1,24 @@ +#include "sequence_facade_benchmarks.h" + +namespace { +const bool registered = [] { + // At 512 MiB useful uint64_t data the default untagged navigation is + // plausibly larger than a 16 MiB shared L3, with total requested storage + // below 1 GiB. The actual node-byte check, not this expectation, decides + // whether to run. + benchmark::RegisterBenchmark( + "FacadeNavigationBeyondL3/Raw64_B256_F8_Prefix", + Access, false, true>) + ->ArgName("N") + ->Arg(EnvironmentMiB("PIXIE_FACADE_NAVIGATION_MIB", 512) / + sizeof(std::uint64_t)); + + RegisterLeaf("Bool_B256"); + RegisterLeaf("U16_B256"); + RegisterLeaf("U32_B256"); + RegisterLeaf("U64_B256"); + RegisterLeaf("U64_B128"); + RegisterLeaf("U64_B512"); + return true; +}(); +} // namespace diff --git a/src/benchmarks/sequence_facade_benchmarks.h b/src/benchmarks/sequence_facade_benchmarks.h new file mode 100644 index 0000000..f30f63a --- /dev/null +++ b/src/benchmarks/sequence_facade_benchmarks.h @@ -0,0 +1,632 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Shared benchmark-local workloads. Do NOT enable PIXIE_SEQUENCE_TREE_TESTING. +// Allocation events, peak requested bytes and payload move counts belong in a +// separate untimed diagnostic, not instrumented versions of these timing +// instantiations. memory_usage_bytes() reports final live requested bytes, not +// allocator/RSS or construction high-water usage. Category counters are +// descriptive, including overlapping subsets such as vector slack and order +// bytes; do not sum them all. +// +// Use pinned, serialized /bench runs, at least five repetitions, under +// /tmp/kilo/pixie-experiment-timing.lock. Select rows rather than running the +// entire registry. CPU ns/iteration is per BATCH for reads/rotations; divide by +// operations_per_iteration for ns/read or ns/rotation. items_per_second already +// uses operation units, except construction, which uses constructed elements. +// Payload access reads only the first uint64_t, NOT the whole payload object. +// +// N always counts elements, not bits or block budgets. Main footprint rows use +// 1/32/128 MiB of useful data (bool: one useful bit), plus a matched N=65536 +// access row for width comparisons. Index16 never exceeds 65536 elements. +// PIXIE_FACADE_LARGE_MIB replaces 128; PIXIE_FACADE_NAVIGATION_MIB replaces +// 512. PIXIE_SEQUENCE_RAM_MIB caps the preflight budget (default cap: 4096 +// MiB); half remains headroom. Linux MemAvailable is only a hint, not a cgroup +// limit. PIXIE_SEQUENCE_L3_MIB is required by the navigation-beyond-L3 row, +// which checks actual UNTAGGED internal-node bytes rather than useful payload +// bytes. + +namespace { + +using pixie::LengthLayout; +using pixie::detail::sequence::PackedValueBlock; +using pixie::detail::sequence::SequenceTree; + +constexpr std::size_t kMiB = 1024 * 1024; +constexpr std::size_t kReads = 64; +constexpr std::size_t kRotations = 128; + +std::uint64_t Mix(std::uint64_t x) { + x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9ULL; + x = (x ^ (x >> 27)) * 0x94d049bb133111ebULL; + return x ^ (x >> 31); +} + +std::size_t EnvironmentMiB(const char* name, std::size_t fallback = 0) { + const char* text = std::getenv(name); + if (text == nullptr || *text < '0' || *text > '9') { + return fallback * kMiB; + } + errno = 0; + char* end = nullptr; + const auto value = std::strtoull(text, &end, 10); + // Bound later conversion to element counts and signed benchmark arguments. + constexpr auto limit = + std::min(std::numeric_limits::max(), + std::numeric_limits::max()) / + (8 * kMiB); + if (errno != 0 || *end != '\0' || value == 0 || value > limit) { + return fallback * kMiB; + } + return static_cast(value) * kMiB; +} + +template +struct Payload { + static_assert(Bytes >= 8 && Bytes % 8 == 0); + std::array words; + explicit Payload(std::size_t i) noexcept { + for (std::size_t j = 0; j < words.size(); ++j) { + words[j] = Mix(i + 42 + j * 0x9e3779b97f4a7c15ULL); + } + } +}; + +struct MoveOnlyPayload : Payload<64> { + using Payload<64>::Payload; + MoveOnlyPayload(const MoveOnlyPayload&) = delete; + MoveOnlyPayload& operator=(const MoveOnlyPayload&) = delete; + MoveOnlyPayload(MoveOnlyPayload&&) noexcept = default; + MoveOnlyPayload& operator=(MoveOnlyPayload&&) noexcept = default; +}; +static_assert(sizeof(Payload<8>) == 8 && sizeof(Payload<64>) == 64 && + sizeof(Payload<256>) == 256 && sizeof(MoveOnlyPayload) == 64); + +template +T Value(std::size_t i) { + if constexpr (std::is_same_v) { + return (Mix(i + 42) & 1) != 0; + } else { + return T(i); + } +} + +template +std::uint64_t Token(const T& value) { + if constexpr (std::is_integral_v) { + return static_cast(value); + } else { + return value.words[0]; + } +} + +template +struct Configuration { + using value_type = T; + static constexpr auto storage_bits = Bits; + static constexpr auto fanout = F; + static constexpr auto layout = Layout; + static constexpr auto useful_bits = + std::is_same_v ? 1 : sizeof(T) * 8; +}; + +template +Block MakeBlock(std::size_t first, std::size_t n) { + using T = typename Block::value_type; + std::array scratch{}; + for (std::size_t i = 0; i < n; ++i) { + scratch[i] = Value(first + i); + } + return Block(std::span(scratch.data(), n)); +} + +template +struct UntaggedVariant : Configuration { + using block_type = PackedValueBlock; + using sequence_type = SequenceTree; + static constexpr bool rebased = false; + static constexpr bool indirect = false; + static constexpr std::size_t chunk_bytes = 0; + static auto Make(std::size_t n) { + const auto blocks = + n / block_type::capacity + (n % block_type::capacity != 0); + auto input = std::views::iota(std::size_t{0}, blocks) | + std::views::transform([=](std::size_t i) { + const auto first = i * block_type::capacity; + return MakeBlock( + first, std::min(block_type::capacity, n - first)); + }); + return sequence_type::from_blocks(input); + } +}; + +template +bool Preflight(benchmark::State& state, + std::size_t n, + bool singletons = false, + std::size_t live_fixtures = 1) { + using T = typename V::value_type; + if constexpr (V::rebased) { + if (n != 0 && n - 1 > std::numeric_limits::max()) { + state.SkipWithError("Permutation index domain exceeded"); + return false; + } + } + std::ifstream input("/proc/meminfo"); + std::string key, rest; + std::size_t kib = 0, available = 0; + while (input >> key >> kib) { + if (key == "MemAvailable:") { + available = kib * 1024; + break; + } + std::getline(input, rest); + } + const auto explicit_cap = EnvironmentMiB("PIXIE_SEQUENCE_RAM_MIB"); + const auto cap = explicit_cap == 0 ? 4096 * kMiB : explicit_cap; + available = available == 0 ? explicit_cap : std::min(available, cap); + // Planning only: current blocks reserve 128 bits for local bookkeeping. + // Budget an extra cache line per leaf/node for tags/padding, the minimum + // branching factor, 2x occupancy/construction slack, and small-tree preflight + // spares. Floating arithmetic avoids overflow for user-supplied sizes. + // Singleton chunks may each reserve the entire chunk budget. + constexpr auto field_bits = V::indirect ? sizeof(void*) * 8 : V::useful_bits; + constexpr auto capacity = + std::max(1, (V::storage_bits - 128) / field_bits); + const long double leaves = singletons ? n : 1 + n / capacity; + const long double nodes = 1 + leaves / (V::fanout / 2 - 1); + long double estimate = + 128 * 1024 + 2 * (leaves * (V::storage_bits / 8 + 64) + + nodes * (2 * V::fanout * sizeof(void*) + 64)); + if constexpr (V::indirect) { + const auto slots = std::max(1, V::chunk_bytes / sizeof(T)); + const long double chunks = singletons ? n : 1 + n / slots; + estimate += 2 * chunks * (std::max(V::chunk_bytes, sizeof(T)) + 128); + } + estimate *= live_fixtures; + state.counters["available_budget_bytes"] = available; + state.counters["planning_bytes"] = static_cast(estimate); + if (available == 0 || estimate > available / 2) { + state.SkipWithError( + "RAM preflight refused row; review PIXIE_SEQUENCE_RAM_MIB"); + return false; + } + return true; +} + +template +void Memory(benchmark::State& state, + const typename V::sequence_type& sequence) { + const auto memory = sequence.memory_usage(); + const auto bytes = memory.total_bytes; + state.counters["blocks"] = memory.blocks; + state.counters["nodes"] = memory.nodes; + if constexpr (requires { memory.tag_padding_bytes; }) { + using Tree = + SequenceTree, + V::fanout, V::layout, true>; + state.counters["tree_block_bytes"] = + memory.blocks * Tree::block_storage_bytes; + state.counters["tree_node_bytes"] = memory.nodes * Tree::node_storage_bytes; + state.counters["payload_capacity_bytes"] = memory.payload_capacity_bytes; + state.counters["block_metadata_bytes"] = memory.block_metadata_bytes; + state.counters["ordering_tree_bytes"] = memory.ordering_tree_bytes; + state.counters["tag_padding_bytes"] = memory.tag_padding_bytes; + state.counters["facade_bytes"] = memory.facade_bytes; + } else if constexpr (requires { memory.chunk_header_bytes; }) { + state.counters["tree_block_bytes"] = memory.tree_block_bytes; + state.counters["tree_node_bytes"] = memory.tree_node_bytes; + state.counters["order_bytes"] = memory.order_bytes; + state.counters["packed_capacity_bits"] = memory.packed_capacity_bits; + state.counters["chunks"] = memory.chunks; + state.counters["chunk_header_bytes"] = memory.chunk_header_bytes; + state.counters["vector_capacity_bytes"] = memory.vector_capacity_bytes; + state.counters["vector_live_bytes"] = memory.vector_live_bytes; + state.counters["vector_slack_bytes"] = memory.vector_slack_bytes; + state.counters["facade_bytes"] = memory.facade_bytes; + } else { + state.counters["tree_block_bytes"] = memory.block_bytes; + state.counters["tree_node_bytes"] = memory.node_bytes; + } + state.counters["elements"] = sequence.size(); + state.counters["useful_bytes"] = + static_cast(sequence.size()) * V::useful_bits / 8; + state.counters["owned_bytes"] = bytes; + state.counters["bytes_per_element"] = + sequence.empty() ? 0 : static_cast(bytes) / sequence.size(); + // sizeof is a separate descriptive counter, NOT an extra summand of bytes. + state.counters["sizeof_facade_or_tree"] = sizeof(sequence); + state.counters["block_budget_bytes"] = V::storage_bits / 8; + state.counters["chunk_budget_bytes"] = V::chunk_bytes; + state.counters["sizeof_value"] = sizeof(typename V::value_type); + state.counters["useful_bits_per_element"] = V::useful_bits; + state.counters["fanout"] = V::fanout; + state.counters["cumulative_lengths"] = V::layout == LengthLayout::cumulative; + state.counters["rebased_merge"] = V::rebased; + state.counters["indirect_storage"] = V::indirect; +} + +template +void Access(benchmark::State& state) { + const std::size_t n = state.range(0); + if (!Preflight(state, n)) { + return; + } + const auto l3 = EnvironmentMiB("PIXIE_SEQUENCE_L3_MIB"); + if constexpr (BeyondL3) { + if (l3 == 0) { + state.SkipWithError( + "Declare measured shared L3 via PIXIE_SEQUENCE_L3_MIB"); + return; + } + } + const auto sequence = V::Make(n); + Memory(state, sequence); + if constexpr (BeyondL3) { + // Only instantiated for the existing untagged tree introspection API. + const auto internal = sequence.internal_memory_bytes(); + state.counters["internal_bytes"] = internal; + state.counters["declared_l3_bytes"] = l3; + state.counters["internal_over_l3"] = static_cast(internal) / l3; + if (internal <= l3) { + state.SkipWithError( + "Navigation does not exceed L3; increase navigation MiB"); + return; + } + } + state.SetLabel( + Dependent + ? "64 dependent reads; scalar/first word only; hash included" + : "64 independent reads; scalar/first word only; hash included"); + // Fresh full-domain positions, not a small repeated query pool. A payload + // reference is never copied merely to read its first word. + std::uint64_t counter = 123, previous = 0; + for (auto _ : state) { + for (std::size_t i = 0; i < kReads; ++i) { + auto key = counter++; + if constexpr (Dependent) { + key ^= previous * 0x9e3779b97f4a7c15ULL; + } + auto value = Token(sequence[Mix(key) % n]); + benchmark::DoNotOptimize(value); + if constexpr (Dependent) { + previous = value; + } + } + } + state.counters["operations_per_iteration"] = kReads; + state.counters["observed_bits_per_read"] = + std::min(64, V::useful_bits); + state.SetItemsProcessed(state.iterations() * kReads); +} + +enum class Rotation { Whole, Global, Local }; +struct Query { + std::size_t left, right, distance; +}; + +template +auto RotationQueries(std::size_t n) { + std::array queries; + for (std::size_t i = 0; i < queries.size(); ++i) { + const auto key = Mix(i + 128); + const auto length = Mode == Rotation::Whole ? n + : Mode == Rotation::Local ? std::min(65, n) + : n / 2 + key % (n / 4); + const auto left = Mix(key) % (n - length + 1); + queries[i] = {left, left + length, 1 + Mix(key + 1) % (length - 1)}; + } + return queries; +} + +template +void RotateBatch(benchmark::State& state) { + using Sequence = typename V::sequence_type; + const std::size_t n = state.range(0); + if (!Preflight(state, n)) { + return; + } + const auto queries = RotationQueries(n); + Sequence sequence; + state.SetLabel( + "128 rotations/batch; fixed stream; rebuild/destruction excluded; " + "rebuild pre-touches data; allocation+repair included"); + for (auto _ : state) { + state.PauseTiming(); + sequence = Sequence{}; + sequence = V::Make(n); + state.ResumeTiming(); + for (const auto& q : queries) { + sequence.rotate_left(q.left, q.right, q.distance); + benchmark::ClobberMemory(); + } + benchmark::DoNotOptimize(sequence); + } + if (sequence.size() != n) { + state.SkipWithError("Rotation changed size"); + return; + } + // Bounded inverse-position oracle; no complete reference input allocation. + const auto matches = [&](std::size_t position) { + auto original = position; + for (auto q = queries.rbegin(); q != queries.rend(); ++q) { + if (original >= q->left && original < q->right) { + original = + q->left + (original - q->left + q->distance) % (q->right - q->left); + } + } + return Token(sequence[position]) == + Token(Value(original)); + }; + for (std::size_t probe = 0; probe < 8; ++probe) { + if (!matches(Mix(probe + 19) % n)) { + state.SkipWithError( + "Rotation disagrees with sampled inverse-position oracle"); + return; + } + } + // Follow one affected output position from every operation through later + // rotations. Uniform probes alone almost always miss the short local ranges. + for (std::size_t i = 0; i < queries.size(); ++i) { + auto position = queries[i].left; + for (std::size_t j = i + 1; j < queries.size(); ++j) { + const auto& q = queries[j]; + if (position >= q.left && position < q.right) { + const auto offset = position - q.left; + const auto length = q.right - q.left; + position = + q.left + (offset >= q.distance ? offset - q.distance + : length - (q.distance - offset)); + } + } + if (!matches(position)) { + state.SkipWithError("Rotation disagrees at an affected position"); + return; + } + } + Memory(state, sequence); + state.counters["operations_per_iteration"] = kRotations; + state.counters["query_pool_bytes"] = sizeof(queries); + state.SetItemsProcessed(state.iterations() * kRotations); +} + +template +void Merge(benchmark::State& state) { + using Sequence = typename V::sequence_type; + const std::size_t n = state.range(0); + // Keep the same batch count for equal logical representations, with at least + // two independent merges per timing interval and bounded fixture storage. + constexpr std::size_t max_batch = 8; + const auto useful_bytes = static_cast(n) * V::useful_bits / 8; + const std::size_t batch = static_cast(std::clamp( + 64.0L * kMiB / useful_bytes, 2.0L, static_cast(max_batch))); + if (!Preflight(state, n, false, batch)) { + return; + } + const auto donor_n = Equal ? n / 2 : n / 64; + const auto receiver_n = n - donor_n; + std::array receivers, donors; + state.SetLabel( + V::rebased + ? "batched rebased merges; independent rotated identities built " + "untimed" + : "batched unchanged-value merges; independent rotated ranges built " + "untimed"); + for (auto _ : state) { + state.PauseTiming(); + for (std::size_t i = 0; i < batch; ++i) { + receivers[i] = Sequence{}; + donors[i] = Sequence{}; + receivers[i] = V::Make(receiver_n); + donors[i] = V::Make(donor_n); + receivers[i].rotate_left(0, receiver_n, receiver_n / 3); + donors[i].rotate_left(0, donor_n, donor_n / 3); + } + state.ResumeTiming(); + for (std::size_t i = 0; i < batch; ++i) { + receivers[i].merge(donors[i]); + benchmark::DoNotOptimize(receivers[i]); + benchmark::ClobberMemory(); + } + } + auto donor_first = Token(Value(donor_n / 3)); + if constexpr (V::rebased) { + donor_first += receiver_n; + } + std::size_t fixture_bytes = 0; + for (std::size_t i = 0; i < batch; ++i) { + const auto& receiver = receivers[i]; + if (receiver.size() != n || !donors[i].empty() || + Token(receiver[0]) != + Token(Value(receiver_n / 3)) || + Token(receiver[receiver_n]) != donor_first) { + state.SkipWithError("Merge size/donor/value invariant failed"); + return; + } + fixture_bytes += + receiver.memory_usage_bytes() + donors[i].memory_usage_bytes(); + } + Memory(state, receivers[0]); + state.counters["active_fixture_bytes"] = fixture_bytes; + state.counters["receiver_elements"] = receiver_n; + state.counters["donor_elements"] = donor_n; + state.counters["operations_per_iteration"] = batch; + state.SetItemsProcessed(state.iterations() * batch); +} + +template +void Build(benchmark::State& state) { + using Sequence = typename V::sequence_type; + const std::size_t n = state.range(0); + if (!Preflight(state, n, Singletons)) { + return; + } + state.SetLabel( + Singletons + ? "singleton construction+merge; unchanged repeated values unless " + "rebased; final destruction excluded" + : "streaming value generation+build; final destruction excluded"); + bool reported = false; + for (auto _ : state) { + auto sequence = [&] { + if constexpr (Singletons) { + Sequence result; + for (std::size_t i = 0; i < n; ++i) { + auto donor = V::Make(1); + result.merge(donor); + } + return result; + } else { + return V::Make(n); + } + }(); + benchmark::DoNotOptimize(sequence); + benchmark::ClobberMemory(); + state.PauseTiming(); + if (!reported) { + Memory(state, sequence); + reported = true; + } + const auto expected_last = Singletons && !V::rebased ? 0 : n - 1; + const bool valid = sequence.size() == n && + Token(sequence[n - 1]) == + Token(Value(expected_last)); + sequence = Sequence{}; + state.ResumeTiming(); + if (!valid) { + state.SkipWithError("Construction size/value invariant failed"); + break; + } + } + state.counters["operations_per_iteration"] = n; + state.SetItemsProcessed(state.iterations() * n); + state.SetBytesProcessed(state.iterations() * (n * V::useful_bits / 8)); +} + +template +void HotLeaf(benchmark::State& state) { + const auto n = Block::capacity - 3; + auto block = MakeBlock(0, n); + const auto queries = RotationQueries(n); + state.SetLabel(Rotate + ? "128 hot-leaf rotations; rebuild excluded; no tree/tags" + : "64 hot-leaf reads; hash included; no tree/tags"); + std::uint64_t counter = 123; + for (auto _ : state) { + if constexpr (Rotate) { + state.PauseTiming(); + block = MakeBlock(0, n); + state.ResumeTiming(); + for (const auto& q : queries) { + block.rotate_left(q.left, q.right, q.distance); + benchmark::ClobberMemory(); + } + benchmark::DoNotOptimize(block); + } else { + for (std::size_t i = 0; i < kReads; ++i) { + auto value = block[Mix(counter++) % n]; + benchmark::DoNotOptimize(value); + } + } + } + state.counters["elements"] = n; + state.counters["block_capacity_elements"] = Block::capacity; + state.counters["sizeof_block"] = sizeof(Block); + state.counters["operations_per_iteration"] = Rotate ? kRotations : kReads; + state.SetItemsProcessed(state.iterations() * (Rotate ? kRotations : kReads)); +} + +// Only the default variants get the whole workload set. Secondary value/storage +// axes get reads, global rotation and construction, with merge comparisons for +// every payload size and index width. Layout knobs get reads and global +// rotation at 32 MiB only. This is a screen, not a Cartesian product. +template +void Sizes(benchmark::internal::Benchmark* row, bool matched = false) { + row->ArgName("N"); + using T = typename V::value_type; + if constexpr (std::is_same_v) { + row->Arg(4096)->Arg(65536); + } else { + if (matched) { + row->Arg(65536); + } + row->Arg(kMiB * 8 / V::useful_bits); + row->Arg(32 * kMiB * 8 / V::useful_bits); + const auto large = EnvironmentMiB("PIXIE_FACADE_LARGE_MIB", 128); + if (large != kMiB && large != 32 * kMiB) { + row->Arg(large * 8 / V::useful_bits); + } + } +} + +template +void Register(const char* variant) { + const auto add = [=](const char* operation, auto function, bool fixed = false, + bool matched = false, std::int64_t iterations = 8) { + const auto name = std::string(operation) + "/" + variant; + auto* row = benchmark::RegisterBenchmark(name.c_str(), function); + if constexpr (Control) { + row->ArgName("N")->Arg(32 * kMiB * 8 / V::useful_bits); + } else { + Sizes(row, matched); + } + // Avoid adaptive millions of untimed full reconstructions for a fast merge. + // Each repetition starts with the same operations/topology, including all + // rotation batches. Increase repetitions for marginal comparisons. + if (fixed) { + row->Iterations(iterations); + } + }; + add("FacadeAccessIndependent", Access, false, true); + constexpr bool is_bool = std::is_same_v; + add("FacadeRotateGlobalBatch", RotateBatch, true, + is_bool); + if constexpr (!Control) { + add("FacadeBuild", Build, false, is_bool); + } + if constexpr (Full) { + add("FacadeAccessDependent", Access); + add("FacadeRotateWholeBatch", RotateBatch, true); + add("FacadeRotateLocal65Batch", RotateBatch, true); + const auto name = std::string("FacadeSingletonBuildMerge/") + variant; + benchmark::RegisterBenchmark(name.c_str(), Build) + ->ArgName("N") + ->Arg(256) + ->Arg(4096) + ->Iterations(8); + } + if constexpr (!Control && (Full || V::indirect || V::rebased)) { + add("FacadeMergeEqual", Merge, true, true, 32); + add("FacadeMergeUnequal", Merge, true, true, 32); + } +} + +template +void RegisterLeaf(const char* variant) { + using Block = PackedValueBlock; + const auto read = std::string("FacadeLeafRead/") + variant; + const auto rotate = std::string("FacadeLeafRotateBatch/") + variant; + benchmark::RegisterBenchmark(read.c_str(), HotLeaf); + benchmark::RegisterBenchmark(rotate.c_str(), HotLeaf); +} + +} // namespace diff --git a/src/benchmarks/sequence_tree_kernels_benchmarks.cpp b/src/benchmarks/sequence_tree_kernels_benchmarks.cpp new file mode 100644 index 0000000..aaab578 --- /dev/null +++ b/src/benchmarks/sequence_tree_kernels_benchmarks.cpp @@ -0,0 +1,97 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using SelectFunction = std::size_t (*)(std::span, + std::size_t) noexcept; + +void NodeSelect(benchmark::State& state, SelectFunction select) { + const auto count = static_cast(state.range(1)); + struct alignas(64) Node { + std::array ends; + }; + struct Query { + std::size_t node; + std::size_t index; + }; + // Shared by both function-pointer controls, not separate template statics: + // matching rows use identical input addresses and deterministic query pools. + static std::array nodes; + static std::array queries; + constexpr auto max = std::numeric_limits::max(); + constexpr auto top = std::size_t{1} + << (std::numeric_limits::digits - 1); + std::mt19937_64 random(0x5e1ec7); + for (std::size_t n = 0; n < nodes.size(); ++n) { + auto end = n % 3 == 0 ? 0 : n % 3 == 1 ? top - 256 : max - 4096; + for (std::size_t i = 0; i < count; ++i) { + end += 1 + random() % 128; + nodes[n].ends[i] = end; + } + } + for (std::size_t i = 0; i < queries.size(); ++i) { + auto& q = queries[i]; + q.node = random() % nodes.size(); + const auto& ends = nodes[q.node].ends; + const auto child = (i / 3) % (count + 1); + // Cycle all child positions, including the implicit last child, with + // before/equal/after endpoints in low, sign-crossing and near-SIZE_MAX + // data. + q.index = child == count ? ends[count - 1] + 1 + random() % 64 + : ends[child] - 1 + i % 3; + if (i % 64 == 62) { + q.index = 0; + } else if (i % 64 == 63) { + q.index = max; + } + } + for (const auto& q : queries) { + const std::span ends(nodes[q.node].ends.data(), count); + const auto expected = static_cast( + std::upper_bound(ends.begin(), ends.end(), q.index) - ends.begin()); + if (select(ends, q.index) != expected) { + state.SkipWithError("Node selection disagrees with upper_bound"); + return; + } + } + + constexpr std::int64_t batch = 64; + std::size_t cursor = 0; + for (auto _ : state) { + for (std::int64_t i = 0; i < batch; ++i) { + const auto& q = queries[cursor++ % queries.size()]; + auto result = select({nodes[q.node].ends.data(), count}, q.index); + benchmark::DoNotOptimize(result); + } + } + state.SetItemsProcessed(state.iterations() * batch); + state.SetLabel( + "hot kernel; 64 queries/iteration; timed: query/node lookup, " + "indirect call, result barrier; excludes setup/validation"); +} + +const bool node_select_registered = [] { + const auto add = [](const char* name, SelectFunction select) { + auto* benchmark = benchmark::RegisterBenchmark(name, NodeSelect, select); + benchmark->ArgNames({"fanout", "ends"}); + for (const int fanout : {4, 8, 16}) { + for (const int count : {fanout / 2 - 1, fanout - 2, fanout - 1}) { + benchmark->Args({fanout, count}); + } + } + }; + add("NodeSelect/Scalar", pixie::detail::sequence::node_select_scalar); + add("NodeSelect/Dispatched", pixie::detail::sequence::node_select); + return true; +}(); + +} // namespace diff --git a/src/tests/bit_sequence_tests.cpp b/src/tests/bit_sequence_tests.cpp new file mode 100644 index 0000000..5da04be --- /dev/null +++ b/src/tests/bit_sequence_tests.cpp @@ -0,0 +1,1463 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pixie; +using namespace pixie::detail::sequence; +using namespace pixie::experimental; + +// No bit operations or exposed writable references: verifies the generic +// contract independently of packed payload assumptions. +struct IntegerBlock { + using value_type = std::uint32_t; + static constexpr std::size_t capacity = 8; + std::array values{}; + std::size_t n = 0; + inline static std::size_t copied_elements = 0; + IntegerBlock() noexcept = default; + explicit IntegerBlock(std::span input) : n(input.size()) { + assert(n <= capacity); + std::copy(input.begin(), input.end(), values.begin()); + } + std::size_t size() const noexcept { return n; } + value_type operator[](std::size_t i) const { + assert(i < n); + return values[i]; + } + void rotate_left(std::size_t l, std::size_t r, std::size_t d) noexcept { + assert(l <= r && r <= n); + if (l != r) { + std::rotate(values.begin() + l, values.begin() + l + d % (r - l), + values.begin() + r); + } + } + void redistribute(IntegerBlock& rhs, std::size_t left) noexcept { + assert(this != &rhs && left <= capacity && left <= n + rhs.n && + n + rhs.n - left <= capacity); + std::array scratch; + const auto total = n + rhs.n; + std::copy_n(values.begin(), n, scratch.begin()); + std::copy_n(rhs.values.begin(), rhs.n, scratch.begin() + n); + std::copy_n(scratch.begin(), left, values.begin()); + std::copy_n(scratch.begin() + left, total - left, rhs.values.begin()); + n = left; + rhs.n = total - left; + copied_elements += total; + } +}; +static_assert(SequenceBlock); +static_assert(SequenceBlock>); +static_assert(SequenceBlock>); +static_assert(SequenceBlock>); + +struct MoveOnlyIntegerBlock : IntegerBlock { + using IntegerBlock::IntegerBlock; + MoveOnlyIntegerBlock() noexcept = default; + MoveOnlyIntegerBlock(const MoveOnlyIntegerBlock&) = delete; + MoveOnlyIntegerBlock& operator=(const MoveOnlyIntegerBlock&) = delete; + MoveOnlyIntegerBlock(MoveOnlyIntegerBlock&&) noexcept = default; + MoveOnlyIntegerBlock& operator=(MoveOnlyIntegerBlock&&) noexcept = default; +}; +static_assert(SequenceBlock); + +struct OddIntegerBlock : IntegerBlock { + using IntegerBlock::IntegerBlock; + static constexpr std::size_t capacity = 7; +}; +static_assert(SequenceBlock); + +struct alignas(128) ConstOnlyAlignedBlock : IntegerBlock { + using IntegerBlock::IntegerBlock; + using IntegerBlock::operator[]; + value_type operator[](std::size_t) = delete; +}; +static_assert(SequenceBlock); + +template +void rotate(std::vector& values, + std::size_t l, + std::size_t r, + std::size_t d) { + if (l != r) { + std::rotate(values.begin() + l, values.begin() + l + d % (r - l), + values.begin() + r); + } +} + +std::vector representative_origins(std::size_t size) { + assert(size != 0); + std::vector origins = { + 0, + 1, + 2, + 63, + 64, + 65, + 127, + 128, + 129, + size / 2, + size > 1 ? size - 2 : 0, + size - 1, + }; + for (const auto numerator : {3U, 5U, 7U, 9U, 11U, 13U, 15U}) { + origins.push_back(size * numerator / 16); + } + std::erase_if(origins, [size](std::size_t origin) { return origin >= size; }); + std::ranges::sort(origins); + origins.erase(std::unique(origins.begin(), origins.end()), origins.end()); + return origins; +} + +std::vector pack(const std::vector& bits) { + std::vector words((bits.size() + 63) / 64); + for (std::size_t i = 0; i < bits.size(); ++i) { + words[i / 64] |= std::uint64_t{bits[i]} << (i % 64); + } + if (bits.size() % 64) { + words.back() |= ~std::uint64_t{0} << (bits.size() % 64); + } + return words; +} +template +std::vector data(std::size_t n, + std::size_t seed = 42) { + std::mt19937_64 random(seed); + std::vector result(n); + for (std::size_t i = 0; i < n; ++i) { + if constexpr (std::same_as) { + result[i] = random() & 1; + } else { + result[i] = static_cast(random()); + } + } + return result; +} +template +Tree build(const std::vector& values, + std::size_t chunk = Tree::block_capacity) { + using Block = typename Tree::block_type; + // A lazy, single-pass-compatible producer avoids an auxiliary block buffer. + auto blocks = + std::views::iota(std::size_t{0}, (values.size() + chunk - 1) / chunk) | + std::views::transform([&](std::size_t index) { + const auto begin = index * chunk; + const auto n = std::min(chunk, values.size() - begin); + if constexpr (std::same_as) { + std::vector local(values.begin() + begin, + values.begin() + begin + n); + return Block(pack(local), n); + } else { + return Block(std::span(values).subspan(begin, n)); + } + }); + return Tree::from_blocks(blocks); +} +template +void check(const Tree& tree, + const std::vector& values) { + ASSERT_TRUE(tree.test_validate()); + ASSERT_EQ(tree.size(), values.size()); + ASSERT_EQ(tree.empty(), values.empty()); + for (std::size_t i = 0; i < values.size(); ++i) { + ASSERT_EQ(tree[i], values[i]) << "index=" << i; + } + std::size_t offset = 0; + tree.for_each_block([&](const auto& block) { + for (std::size_t i = 0; i < block.size(); ++i) { + ASSERT_EQ(block[i], values[offset++]); + } + }); + EXPECT_EQ(offset, values.size()); + const auto memory = tree.memory_usage(); + EXPECT_EQ(memory.total_bytes, + sizeof(tree) + memory.block_bytes + memory.node_bytes); + EXPECT_EQ(memory.block_bytes, memory.blocks * Tree::block_storage_bytes); + EXPECT_EQ(memory.node_bytes, memory.nodes * Tree::node_storage_bytes); + EXPECT_EQ(tree.node_count(), memory.nodes); + EXPECT_EQ(tree.internal_memory_bytes(), memory.node_bytes); + EXPECT_LE(memory.blocks, tree.size() / (Tree::block_capacity / 2) + 2); + if (tree.empty()) { + EXPECT_EQ(memory.total_bytes, sizeof(tree)); + } +} + +template +class SequenceTreeSpec : public ::testing::Test {}; +using TreeTypes = ::testing::Types< + SequenceTree, 4>, + SequenceTree, 8, LengthLayout::individual>, + SequenceTree, 16>, + SequenceTree, 4, LengthLayout::individual>, + SequenceTree, 8>, + SequenceTree, 16, LengthLayout::individual>, + SequenceTree, 4>, + SequenceTree, + SequenceTree, + SequenceTree, + SequenceTree, + SequenceTree, + SequenceTree, + SequenceTree>; +TYPED_TEST_SUITE(SequenceTreeSpec, TreeTypes); + +TYPED_TEST(SequenceTreeSpec, EmptyInvalidOwnershipAndAccounting) { + using Tree = TypeParam; + static_assert(!std::is_copy_constructible_v); + static_assert(std::is_nothrow_move_constructible_v); + static_assert(std::is_nothrow_move_assignable_v); + Tree empty; + check(empty, {}); + empty.rotate_left(0, 0, -1); + empty.merge(empty); + EXPECT_THROW(empty[0], std::out_of_range); + EXPECT_THROW(empty.split_off(1), std::out_of_range); + EXPECT_THROW(empty.rotate_left(0, 1, 0), std::out_of_range); + EXPECT_THROW(empty.rotate_left(1, 0, 0), std::out_of_range); + auto expected = data(Tree::block_capacity * 3 + 1); + auto tree = build(expected, 3); + check(tree, expected); + EXPECT_THROW(tree[tree.size()], std::out_of_range); + EXPECT_THROW(tree.split_off(tree.size() + 1), std::out_of_range); + EXPECT_THROW(tree.rotate_left(0, tree.size() + 1, 0), std::out_of_range); + tree.merge(tree); + tree.merge(empty); + auto suffix = tree.split_off(tree.size()); + check(suffix, {}); + auto all = tree.split_off(0); + check(tree, {}); + empty.merge(all); + check(all, {}); + Tree moved(std::move(empty)); + check(empty, {}); + tree = std::move(moved); + check(moved, {}); + auto& alias = tree; + tree = std::move(alias); + check(tree, expected); +} + +TYPED_TEST(SequenceTreeSpec, EveryShortCutAndRotation) { + using Tree = TypeParam; + for (std::size_t n = 0; n <= 9; ++n) { + const auto original = data(n, n); + for (std::size_t p = 0; p <= n; ++p) { + auto tree = build(original); + auto suffix = tree.split_off(p); + check(tree, {original.begin(), original.begin() + p}); + check(suffix, {original.begin() + p, original.end()}); + tree.merge(suffix); + check(tree, original); + } + for (std::size_t l = 0; l <= n; ++l) { + for (std::size_t r = l; r <= n; ++r) { + for (std::size_t d = 0; d <= r - l + 1; ++d) { + auto tree = build(original); + auto expected = original; + tree.rotate_left(l, r, d); + rotate(expected, l, r, d); + check(tree, expected); + } + } + } + } +} + +TYPED_TEST(SequenceTreeSpec, RandomCutsJoinsRotationsAndUnderfullSeams) { + using Tree = TypeParam; + std::mt19937_64 random(781293); + auto expected = data(Tree::block_capacity * 35 + 3); + auto tree = build(expected); + for (std::size_t step = 0; step < 250; ++step) { + SCOPED_TRACE(step); + if (step % 7 == 0) { + const auto p = random() % (tree.size() + 1); + auto suffix = tree.split_off(p); + check(tree, {expected.begin(), expected.begin() + p}); + check(suffix, {expected.begin() + p, expected.end()}); + if (step % 14 == 0) { + suffix.merge(tree); + tree = std::move(suffix); + rotate(expected, 0, expected.size(), p); + } else { + tree.merge(suffix); + } + } else if (step % 11 == 0) { + auto tail = data(1 + random() % 17, step); + auto donor = build(tail); + tree.merge(donor); + expected.insert(expected.end(), tail.begin(), tail.end()); + check(donor, {}); + } else { + auto l = random() % (tree.size() + 1); + auto r = random() % (tree.size() + 1); + if (l > r) { + std::swap(l, r); + } + if (step % 5 == 0) { + l = 1; + r = tree.size() - 1; + } + const auto d = + step % 3 == 0 ? std::numeric_limits::max() : random(); + tree.rotate_left(l, r, d); + rotate(expected, l, r, d); + } + check(tree, expected); + } +} + +TYPED_TEST(SequenceTreeSpec, UnequalHeightsTinyMergesRootBoundaries) { + using Tree = TypeParam; + auto expected = data(Tree::block_capacity * 70 + 1); + auto tree = build(expected); + const auto initial_height = tree.height(); + ASSERT_GT(initial_height, 0); + for (const auto p : + {std::size_t{1}, Tree::block_capacity - 1, Tree::block_capacity * 4, + Tree::block_capacity * 16 + 1, tree.size() - 1}) { + auto suffix = tree.split_off(p); + check(tree, {expected.begin(), expected.begin() + p}); + check(suffix, {expected.begin() + p, expected.end()}); + tree.merge(suffix); + check(tree, expected); + } + auto suffix = tree.split_off(1); + EXPECT_EQ(tree.height(), 0); + tree.merge(suffix); + check(tree, expected); + Tree tiny; + std::vector short_expected; + for (std::size_t i = 0; i < 1100; ++i) { + const auto one = data(1, i); + auto donor = build(one); + if (i % 2) { + tiny.merge(donor); + short_expected.push_back(one[0]); + } else { + donor.merge(tiny); + tiny = std::move(donor); + short_expected.insert(short_expected.begin(), one[0]); + } + ASSERT_TRUE(tiny.test_validate()) << i; + } + check(tiny, short_expected); +} + +template +class BitBlockSpec : public ::testing::Test {}; +using BlockTypes = ::testing::Types, + BitBlock<192>, + BitBlock<>, + PackedBitBlock<512>, + PackedBitBlock<>, + PermutedBitBlock<>, + PermutedBitBlock>; +TYPED_TEST_SUITE(BitBlockSpec, BlockTypes); + +template +void check_block(const Block& b, const std::vector& expected) { + ASSERT_EQ(b.size(), expected.size()); + const auto flat = b.flatten(); + for (std::size_t i = 0; i < Block::capacity; ++i) { + ASSERT_EQ(bool((flat[i / 64] >> (i % 64)) & 1), + i < expected.size() ? expected[i] : false) + << i; + if (i < expected.size()) { + ASSERT_EQ(b[i], expected[i]) << i; + } + } +} + +TYPED_TEST(BitBlockSpec, ExhaustiveShortPatterns) { + using Block = TypeParam; + EXPECT_THROW(Block({}, Block::capacity + 1), std::invalid_argument); + EXPECT_THROW(Block({}, 1), std::invalid_argument); + Block empty; + empty.rotate_left(0, 0, -1); + EXPECT_THROW(empty[0], std::out_of_range); + for (std::size_t n = 0; n <= 6; ++n) { + for (std::uint64_t pattern = 0; pattern < (1ULL << n); ++pattern) { + const auto dirty = pattern | (~std::uint64_t{0} << n); + for (std::size_t l = 0; l <= n; ++l) { + for (std::size_t r = l; r <= n; ++r) { + for (std::size_t d = 0; d <= r - l + 1; ++d) { + Block b(std::span(&dirty, 1), n); + std::vector expected(n); + for (std::size_t i = 0; i < n; ++i) { + expected[i] = (pattern >> i) & 1; + } + b.rotate_left(l, r, d); + rotate(expected, l, r, d); + auto words = pack(expected); + const auto mask = n ? (1ULL << n) - 1 : 0; + EXPECT_EQ(b.flatten()[0], words.empty() ? 0 : words[0] & mask); + for (std::size_t i = 0; i < n; ++i) { + ASSERT_EQ(b[i], expected[i]); + } + } + } + } + } + } +} + +TYPED_TEST(BitBlockSpec, RepresentativeOriginsRedistributionAndDirtyPadding) { + using Block = TypeParam; + std::mt19937_64 random(865124); + for (const std::size_t n : + {std::size_t{65}, Block::capacity - 3, Block::capacity}) { + auto initial = data>(n); + for (const auto origin : representative_origins(n)) { + Block a(pack(initial), n); + auto expected = initial; + a.rotate_left(0, n, origin); + rotate(expected, 0, n, origin); + const auto l = origin % 64; + const auto r = std::min(n, l + 65); + const auto d = random(); + a.rotate_left(l, r, d); + rotate(expected, l, r, d); + Block b; + const auto cut = origin % (n + 1); + a.redistribute(b, cut); + check_block(a, {expected.begin(), expected.begin() + cut}); + check_block(b, {expected.begin() + cut, expected.end()}); + a.redistribute(b, n); + check_block(a, expected); + check_block(b, {}); + } + } + // Representative origins cover word and 128-bit chunk boundaries; randomized + // cases cover arbitrary block sizes and redistribution cuts without making + // the coverage configuration exhaust every physical origin. + for (std::size_t step = 0; step < 128; ++step) { + const auto n = random() % (Block::capacity + 1); + const auto m = random() % (Block::capacity + 1); + auto avalues = data>(n, step); + auto bvalues = data>(m, step + 1); + Block a(pack(avalues), n), b(pack(bvalues), m); + a.rotate_left(0, n, 65); + b.rotate_left(0, m, 127); + rotate(avalues, 0, n, 65); + rotate(bvalues, 0, m, 127); + avalues.insert(avalues.end(), bvalues.begin(), bvalues.end()); + const auto low = n + m > Block::capacity ? n + m - Block::capacity : 0; + const auto high = std::min(Block::capacity, n + m); + const auto cut = low + random() % (high - low + 1); + a.redistribute(b, cut); + check_block(a, {avalues.begin(), avalues.begin() + cut}); + check_block(b, {avalues.begin() + cut, avalues.end()}); + } +} + +TEST(PermutedBitBlock, MapKernelsPreserveRepresentationAndContents) { + using Block = PermutedBitBlock<>; + static_assert(std::is_trivially_copyable_v); + std::mt19937_64 random(1282048); + for (const std::size_t n : {257, 385, 513, 1921, 2045, 2048}) { + auto expected = data>(n); + Block b(pack(expected), n); + auto apply = [&](std::size_t l, std::size_t r, std::size_t d) { + b.rotate_left(l, r, d); + rotate(expected, l, r, d); + check_block(b, expected); + }; + for (std::size_t l = 0; l <= n / 128; ++l) { + for (std::size_t r = l; r <= n / 128; ++r) { + for (std::size_t d = 0; d <= r - l; ++d) { + apply(l * 128, r * 128, d * 128); + } + } + } + // Whole rotations use varied deterministic distances so the map state sees + // origins across the block without repeating the same branch coverage n + // times. The aligned triples above retain exhaustive map-kernel controls. + for (std::size_t iteration = 0; iteration < 64; ++iteration) { + apply(0, n, 1 + random() % n); + apply(1, n - 1, random()); + apply(63, 257, 65); + } + Block::Payload words; + words.fill(~std::uint64_t{0}); + Block uniform(words, n); + uniform.rotate_left(0, 256, 128); + uniform.rotate_left(0, n, 127); + std::array before, after; + std::memcpy(before.data(), &uniform, sizeof(uniform)); + uniform.rotate_left(1, n - 1, -1); + uniform.rotate_left(63, 257, 65); + std::memcpy(after.data(), &uniform, sizeof(uniform)); + EXPECT_EQ(before, after); + } +} + +TEST(PermutedBitBlock, WrappedMappedRedistributionSmoke) { + using Block = PermutedBitBlock<>; + auto expected = data>(1921); + Block a(pack(expected), expected.size()); + for (const auto h : + {std::size_t{0}, std::size_t{1}, std::size_t{128}, std::size_t{1900}}) { + a.rotate_left(0, 512, 128); + rotate(expected, 0, 512, 128); + a.rotate_left(0, a.size(), h); + rotate(expected, 0, expected.size(), h); + a.rotate_left(63, 257, 65); + rotate(expected, 63, 257, 65); + Block b; + a.redistribute(b, 127); + check_block(a, {expected.begin(), expected.begin() + 127}); + check_block(b, {expected.begin() + 127, expected.end()}); + a.redistribute(b, expected.size()); + check_block(a, expected); + } +} + +TEST(SequenceTreeSeam, TwoTinyLeavesRequireTheirNeighbors) { + using Tree = SequenceTree; + for (std::size_t left_count : {1, 9, 33, 129}) { + for (std::size_t right_count : {8, 16, 40, 136}) { + auto avalues = data(left_count, left_count); + auto bvalues = data(right_count, right_count); + auto a = build(avalues); + auto b = build(bvalues); + auto tail = b.split_off(7); + b = std::move(tail); + bvalues.erase(bvalues.begin(), bvalues.begin() + 7); + a.merge(b); + avalues.insert(avalues.end(), bvalues.begin(), bvalues.end()); + check(a, avalues); + check(b, {}); + } + } +} + +TEST(SequenceTreeLayout, TypedAlignedPhysicalBudgets) { + static_assert(sizeof(PackedBitBlock<512>) == 64); + static_assert(sizeof(PackedBitBlock<1024>) == 128); + static_assert(sizeof(PackedBitBlock<2048>) == 256); + static_assert(sizeof(PackedBitBlock<4096>) == 512); + static_assert(PackedBitBlock<2048>::capacity == 1920); + static_assert(alignof(PackedBitBlock<>) == alignof(pixie::CacheLine)); + static_assert(BitBlock<>::capacity == 2048); + static_assert(sizeof(BitBlock<>) == 320); + static_assert(BitBlock<>::payload_offset_bytes == 16); + static_assert(BitBlock<>::payload_alignment == alignof(std::uint64_t)); + EXPECT_EQ((SequenceTree::node_storage_bytes), 64); + EXPECT_EQ((SequenceTree::node_storage_bytes), 128); + EXPECT_EQ((SequenceTree::node_storage_bytes), 256); + using Tree = SequenceTree>; + auto tree = build(data(20000)); + EXPECT_TRUE(tree.test_validate()); + EXPECT_EQ(tree.payload_capacity_bytes(), tree.block_count() * 240); + EXPECT_EQ(tree.metadata_bytes() + tree.payload_capacity_bytes(), + tree.memory_usage_bytes()); +} + +TEST(SequenceTreeLayout, ConstOnlyReadsAndOverAlignedLeaves) { + using Tree = SequenceTree; + static_assert(alignof(ConstOnlyAlignedBlock) == 128); + static_assert(Tree::block_alignment == alignof(ConstOnlyAlignedBlock)); + auto expected = data(257); + auto tree = build(expected); + auto verify = [&] { + check(std::as_const(tree), expected); + std::size_t visited = 0; + tree.for_each_block([&](const ConstOnlyAlignedBlock& block) { + EXPECT_EQ(reinterpret_cast(&block) % 128, 0); + ++visited; + }); + EXPECT_EQ(visited, tree.block_count()); + }; + verify(); + auto tail = tree.split_off(17); + check(tail, {expected.begin() + 17, expected.end()}); + tree.merge(tail); + verify(); + tree.rotate_left(1, tree.size() - 1, 73); + rotate(expected, 1, expected.size() - 1, 73); + verify(); +} + +TEST(SequenceTreeConstruction, SinglePassAndThrowingProducer) { + using Tree = SequenceTree; + std::istringstream stream("1 2 3 4 5 6 7 8 9 10 11 12 13"); + auto blocks = std::ranges::istream_view(stream) | + std::views::transform([](std::uint32_t n) { + return IntegerBlock(std::span(&n, 1)); + }); + auto tree = Tree::from_blocks(blocks); + check(tree, {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}); + const auto live = Tree::test_counters.live_allocations; + auto throwing = std::views::iota(0, 100) | std::views::transform([](int n) { + if (n == 47) { + throw std::runtime_error("producer"); + } + const auto value = static_cast(n); + return IntegerBlock(std::span(&value, 1)); + }); + EXPECT_THROW(Tree::from_blocks(throwing), std::runtime_error); + EXPECT_EQ(Tree::test_counters.live_allocations, live); +} + +TEST(SequenceTreeConstruction, StreamingBuildHasLinearAllocationCount) { + using Tree = SequenceTree; + for (const std::size_t leaves : {1, 64, 4097, 65536}) { + auto blocks = std::views::iota(std::size_t{0}, leaves) | + std::views::transform([](std::size_t index) { + std::array values; + for (std::size_t i = 0; i < values.size(); ++i) { + values[i] = index * IntegerBlock::capacity + i; + } + return IntegerBlock(values); + }); + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + auto tree = Tree::from_blocks(blocks); + const auto counters = Tree::test_counters; + std::size_t carry_nodes = 0; + std::size_t levels = 1; + for (auto remaining = leaves / 8; remaining; remaining /= 8) { + carry_nodes += remaining; + ++levels; + } + // One leaf allocation per block, geometric bottom-up carries, and just + // one height-bounded spare pool for the final fringe, not per-leaf merges. + EXPECT_LE(counters.allocations, + leaves + carry_nodes + (leaves > 1 ? 3 * levels : 0)); + EXPECT_LE(counters.node_visits, 4 * levels); + EXPECT_LE(counters.child_transfers, 4 * leaves + 100 * levels); + EXPECT_LE(IntegerBlock::copied_elements, + 2 * leaves * IntegerBlock::capacity); + EXPECT_EQ(tree.size(), leaves * IntegerBlock::capacity); + EXPECT_EQ(tree.block_count(), leaves); + ASSERT_TRUE(tree.test_validate()); + std::size_t expected = 0; + tree.for_each_block([&](const IntegerBlock& block) { + for (std::size_t i = 0; i < block.size(); ++i) { + ASSERT_EQ(block[i], expected++); + } + }); + EXPECT_EQ(expected, tree.size()); + } +} + +template +void every_preflight_allocation_is_strong_and_leak_free() { + const auto original = data(153); + const auto donor_values = data(35, 21); + for (int operation = 0; operation < 4; ++operation) { + std::size_t allocations = 0; + { + auto tree = build(original); + auto donor = build(donor_values); + // Create underfull leaves at both joining ends, including a third + // neighbor. + if (operation == 1) { + auto discarded = tree.split_off(145); + auto tail = donor.split_off(7); + donor = std::move(tail); + } + Tree::test_reset_counters(); + if (operation == 0) { + auto tail = tree.split_off(73); + } else if (operation == 1) { + tree.merge(donor); + } else if (operation == 2) { + tree.rotate_left(1, 151, 71); + } else { + tree.rotate_left(0, tree.size(), 71); + } + allocations = Tree::test_counters.allocations; + ASSERT_GT(allocations, 0); + } + for (std::size_t failure = 0; failure <= allocations; ++failure) { + SCOPED_TRACE(::testing::Message() << operation << ":" << failure); + auto tree = build(original); + auto donor = build(donor_values); + auto expected = original; + auto donor_expected = donor_values; + if (operation == 1) { + auto discarded = tree.split_off(145); + expected.resize(145); + auto tail = donor.split_off(7); + donor = std::move(tail); + donor_expected.erase(donor_expected.begin(), + donor_expected.begin() + 7); + } + const auto identities = tree.test_leaf_identities(); + const auto donor_identities = donor.test_leaf_identities(); + const auto node_identities = tree.test_internal_node_identities(); + const auto donor_node_identities = donor.test_internal_node_identities(); + const auto live = Tree::test_counters.live_allocations; + const auto live_bytes = Tree::test_counters.live_bytes; + Tree tail; + auto apply = [&] { + if (operation == 0) { + tail = tree.split_off(73); + } else if (operation == 1) { + tree.merge(donor); + } else if (operation == 2) { + tree.rotate_left(1, 151, 71); + } else { + tree.rotate_left(0, tree.size(), 71); + } + }; + Tree::test_reset_counters(); + Tree::test_fail_after(static_cast(failure)); + if (failure < allocations) { + EXPECT_THROW(apply(), std::bad_alloc); + } else { + // All preflight allocations succeed, but any allocation during commit + // would fail. This exercises the nonallocating commit with recycling. + EXPECT_NO_THROW(apply()); + } + Tree::test_fail_after(-1); + if (failure < allocations) { + EXPECT_EQ(Tree::test_counters.live_allocations, live); + EXPECT_EQ(Tree::test_counters.live_bytes, live_bytes); + EXPECT_EQ(tree.test_leaf_identities(), identities); + EXPECT_EQ(donor.test_leaf_identities(), donor_identities); + EXPECT_EQ(tree.test_internal_node_identities(), node_identities); + EXPECT_EQ(donor.test_internal_node_identities(), donor_node_identities); + } else { + EXPECT_EQ(Tree::test_counters.allocations, allocations); + if (operation == 0) { + check(tail, {expected.begin() + 73, expected.end()}); + expected.resize(73); + } else if (operation == 1) { + expected.insert(expected.end(), donor_expected.begin(), + donor_expected.end()); + donor_expected.clear(); + } else if (operation == 2) { + rotate(expected, 1, 151, 71); + } else { + rotate(expected, 0, expected.size(), 71); + } + } + check(tree, expected); + check(donor, donor_expected); + } + } + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TEST(SequenceTreeFailure, ConstructionFailureReclaimsPartialLevels) { + using Tree = SequenceTree; + const auto expected = data(511); + Tree::test_reset_counters(); + { auto tree = build(expected); } + const auto allocations = Tree::test_counters.allocations; + for (std::size_t i = 0; i < allocations; ++i) { + Tree::test_fail_after(i); + EXPECT_THROW(build(expected), std::bad_alloc); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.live_allocations, 0); + } +} + +TEST(SequenceTreeFailure, NoRepairMergeUsesHeightGapBudgetAndRollsBack) { + using Tree = SequenceTree; + const std::array, 6> shapes{ + {{36, 36}, {512, 8}, {8, 512}, {512, 32}, {32, 512}, {512, 512}}}; + for (const auto& [left_size, right_size] : shapes) { + SCOPED_TRACE(::testing::Message() << left_size << ":" << right_size); + const auto left_values = data(left_size, 13); + const auto right_values = data(right_size, 17); + std::size_t budget = 0; + { + auto tree = build(left_values); + auto donor = build(right_values); + const auto taller = std::max(tree.height(), donor.height()); + const auto gap = taller - std::min(tree.height(), donor.height()); + budget = gap + 1; + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + tree.merge(donor); + EXPECT_EQ(Tree::test_counters.allocations, budget); + EXPECT_LT(Tree::test_counters.allocations, 64 * (taller + 2)); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(IntegerBlock::copied_elements, 0); + auto expected = left_values; + expected.insert(expected.end(), right_values.begin(), right_values.end()); + check(tree, expected); + check(donor, {}); + } + for (std::size_t failure = 0; failure < budget; ++failure) { + SCOPED_TRACE(failure); + auto tree = build(left_values); + auto donor = build(right_values); + const auto leaves = tree.test_leaf_identities(); + const auto donor_leaves = donor.test_leaf_identities(); + const auto nodes = tree.test_internal_node_identities(); + const auto donor_nodes = donor.test_internal_node_identities(); + const auto live = Tree::test_counters.live_allocations; + const auto live_bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(static_cast(failure)); + EXPECT_THROW(tree.merge(donor), std::bad_alloc); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.live_allocations, live); + EXPECT_EQ(Tree::test_counters.live_bytes, live_bytes); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(donor.test_leaf_identities(), donor_leaves); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + EXPECT_EQ(donor.test_internal_node_identities(), donor_nodes); + check(tree, left_values); + check(donor, right_values); + } + } + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TEST(SequenceTreeFailure, LeafRootMergeAllocatesOnlyRequiredParent) { + using Tree = SequenceTree; + const std::array, 8> shapes{ + {{1, 1}, {1, 7}, {4, 4}, {3, 5}, {8, 8}, {8, 1}, {1, 8}, {4, 5}}}; + for (const auto& [left_size, right_size] : shapes) { + SCOPED_TRACE(::testing::Message() << left_size << ":" << right_size); + const auto left_values = data(left_size, 31); + const auto right_values = data(right_size, 37); + const std::size_t budget = left_size + right_size > Tree::block_capacity; + for (std::size_t failure = 0; failure <= budget; ++failure) { + auto tree = build(left_values); + auto donor = build(right_values); + ASSERT_EQ(tree.height(), 0); + ASSERT_EQ(donor.height(), 0); + const auto leaves = tree.test_leaf_identities(); + const auto donor_leaves = donor.test_leaf_identities(); + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + const auto live = Tree::test_counters.live_allocations; + const auto live_bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(static_cast(failure)); + if (failure < budget) { + EXPECT_THROW(tree.merge(donor), std::bad_alloc); + } else { + // With budget zero, failure injection rejects any attempted allocation. + EXPECT_NO_THROW(tree.merge(donor)); + } + Tree::test_fail_after(-1); + if (failure < budget) { + EXPECT_EQ(Tree::test_counters.live_allocations, live); + EXPECT_EQ(Tree::test_counters.live_bytes, live_bytes); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(donor.test_leaf_identities(), donor_leaves); + check(tree, left_values); + check(donor, right_values); + } else { + EXPECT_EQ(Tree::test_counters.allocations, budget); + if (left_size >= Tree::block_capacity / 2 && + right_size >= Tree::block_capacity / 2 && budget != 0) { + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(IntegerBlock::copied_elements, 0); + } + EXPECT_EQ(tree.height(), budget); + EXPECT_EQ(tree.node_count(), budget); + EXPECT_EQ(tree.block_count(), budget + 1); + auto expected = left_values; + expected.insert(expected.end(), right_values.begin(), + right_values.end()); + check(tree, expected); + check(donor, {}); + } + } + } + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +std::size_t common_identity_count(std::vector before, + std::vector after) { + std::sort(before.begin(), before.end(), std::less{}); + std::sort(after.begin(), after.end(), std::less{}); + EXPECT_EQ(std::adjacent_find(before.begin(), before.end()), before.end()); + EXPECT_EQ(std::adjacent_find(after.begin(), after.end()), after.end()); + std::vector common; + std::set_intersection(before.begin(), before.end(), after.begin(), + after.end(), std::back_inserter(common), + std::less{}); + return common.size(); +} + +TEST(SequenceTreeRecycling, FullSpineJoinRetainsDismantledAllocations) { + using Tree = SequenceTree; + for (const std::size_t donor_leaves : {1, 64}) { + auto expected = data(64 * Tree::block_capacity); + const auto donor_values = + data(donor_leaves * Tree::block_capacity, 19); + auto tree = build(expected); + auto donor = build(donor_values); + auto nodes = tree.test_internal_node_identities(); + const auto donor_nodes = donor.test_internal_node_identities(); + nodes.insert(nodes.end(), donor_nodes.begin(), donor_nodes.end()); + auto leaves = tree.test_leaf_identities(); + const auto donor_ids = donor.test_leaf_identities(); + leaves.insert(leaves.end(), donor_ids.begin(), donor_ids.end()); + const auto budget = tree.height() - donor.height() + 1; + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_reset_counters(); + Tree::test_fail_after(budget); + tree.merge(donor); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, budget); + EXPECT_EQ(Tree::test_counters.peak_bytes - bytes, + budget * Tree::node_storage_bytes); + EXPECT_EQ( + common_identity_count(nodes, tree.test_internal_node_identities()), + nodes.size()); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + expected.insert(expected.end(), donor_values.begin(), donor_values.end()); + check(tree, expected); + check(donor, {}); + } + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +template +class SequenceTreeRecyclingSpec : public ::testing::Test {}; +using RecyclingTreeTypes = + ::testing::Types, + SequenceTree, + SequenceTree, + SequenceTree, + SequenceTree, + SequenceTree>; +TYPED_TEST_SUITE(SequenceTreeRecyclingSpec, RecyclingTreeTypes); + +TYPED_TEST(SequenceTreeRecyclingSpec, + EveryPreflightAllocationIsStrongAndLeakFree) { + every_preflight_allocation_is_strong_and_leak_free(); +} + +TYPED_TEST(SequenceTreeRecyclingSpec, RotationDeficitAndExactLeafSpares) { + using Tree = TypeParam; + const auto original = data(4096 * Tree::block_capacity); + // Whole/partial, aligned/unaligned, and two cuts inside the same leaf. + const std::array, 7> ranges{{ + {0, original.size(), 8}, + {0, original.size(), 9}, + {8, original.size() - 8, 16}, + {1, original.size() - 1, 17}, + {1, 10, 1}, + {0, original.size() - 1, 8}, + {1, original.size(), 7}, + }}; + for (const auto& [left, right, distance] : ranges) { + SCOPED_TRACE(::testing::Message() + << left << ":" << right << ":" << distance); + auto expected = original; + auto tree = build(original); + const bool whole = left == 0 && right == original.size(); + const auto nodes = whole ? 5 * tree.height() + 4 : 15 * tree.height() + 24; + const auto leaves = std::size_t(left % Tree::block_capacity != 0) + + ((left + distance) % Tree::block_capacity != 0) + + (right % Tree::block_capacity != 0); + const auto bytes = Tree::test_counters.live_bytes; + const auto ids = tree.test_leaf_identities(); + Tree::test_reset_counters(); + Tree::test_fail_after(nodes + leaves); + tree.rotate_left(left, right, distance); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, nodes + leaves); + EXPECT_EQ( + Tree::test_counters.peak_bytes - bytes, + nodes * Tree::node_storage_bytes + leaves * Tree::block_storage_bytes); + rotate(expected, left, right, distance); + check(tree, expected); + if (leaves == 0) { + EXPECT_EQ(common_identity_count(ids, tree.test_leaf_identities()), + ids.size()); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + } + } + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TYPED_TEST(SequenceTreeRecyclingSpec, CompleteChildRotationsNeverAllocate) { + using Tree = TypeParam; + /** + * @brief Force partial internal nodes and unequal child totals. + * @details The last leaf has five elements, so every rotated position is + * legal. + */ + auto expected = data(257 * Tree::block_capacity - 3); + auto tree = build(expected); + const auto nodes = tree.test_internal_node_identities(); + const auto leaves = tree.test_leaf_identities(); + const auto shapes = tree.test_child_boundaries(); + for (auto index : {std::size_t{0}, shapes.size() / 2, shapes.size() - 1}) { + const auto& boundaries = shapes[index]; + for (std::size_t a = 0; a + 2 < boundaries.size(); ++a) { + for (std::size_t b = a + 1; b + 1 < boundaries.size(); ++b) { + for (std::size_t c = b + 1; c < boundaries.size(); ++c) { + const auto l = boundaries[a], m = boundaries[b], r = boundaries[c]; + SCOPED_TRACE(::testing::Message() + << index << ':' << l << ':' << m << ':' << r); + Tree::test_reset_counters(); + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(0); + EXPECT_NO_THROW(tree.rotate_left(l, r, m - l)); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_EQ(Tree::test_counters.peak_bytes, bytes); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + if (index == 0) { + // One covering-root visit; only included global endpoints require + // child-spine occupancy reads, without revisiting the root. + EXPECT_EQ( + Tree::test_counters.node_visits, + 1 + (tree.height() - 1) * ((l == 0) + (r == tree.size()))); + } + rotate(expected, l, r, m - l); + check(tree, expected); + EXPECT_EQ(common_identity_count(nodes, + tree.test_internal_node_identities()), + nodes.size()); + EXPECT_EQ(common_identity_count(leaves, tree.test_leaf_identities()), + leaves.size()); + /** @brief The inverse stays child aligned despite unequal totals. */ + Tree::test_fail_after(0); + EXPECT_NO_THROW(tree.rotate_left(l, r, r - m)); + Tree::test_fail_after(-1); + rotate(expected, l, r, r - m); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + check(tree, expected); + } + } + } + } +} + +TYPED_TEST(SequenceTreeRecyclingSpec, ChildBoundaryReadsOnlyNecessarySpines) { + using Tree = TypeParam; + const auto original = data(4096 * Tree::block_capacity); + auto tree = build(original); + const auto shapes = tree.test_child_boundaries(); + const auto leaves = tree.test_leaf_identities(); + const auto nodes = tree.test_internal_node_identities(); + // Root-first preorder starts with the first-child chain down to leaf parents. + for (std::size_t depth = 0; depth < tree.height(); ++depth) { + const auto& boundaries = shapes[depth]; + ASSERT_GE(boundaries.size(), 5); + for (const bool prefix : {false, true}) { + for (const bool suffix : {false, true}) { + const auto left = boundaries[prefix ? 0 : 1]; + const auto right = boundaries[boundaries.size() - (suffix ? 1 : 2)]; + const auto middle = boundaries[2]; + SCOPED_TRACE(::testing::Message() + << depth << ':' << prefix << ':' << suffix); + Tree::test_reset_counters(); + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(0); + EXPECT_NO_THROW(tree.rotate_left(left, right, middle - left)); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.node_visits, + depth + 1 + + (tree.height() - depth - 1) * + ((left == 0) + (right == tree.size()))); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_EQ(Tree::test_counters.peak_bytes, bytes); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + auto expected = original; + rotate(expected, left, right, middle - left); + check(tree, expected); + Tree::test_fail_after(0); + EXPECT_NO_THROW(tree.rotate_left(left, right, right - middle)); + Tree::test_fail_after(-1); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + check(tree, original); + } + } + } +} + +TYPED_TEST(SequenceTreeRecyclingSpec, UnderfullExteriorChildFallsBack) { + using Tree = TypeParam; + for (const bool first : {false, true}) { + for (const bool whole : {false, true}) { + SCOPED_TRACE(::testing::Message() << first << ':' << whole); + auto expected = data(4096 * Tree::block_capacity); + auto tree = build(expected); + if (first) { + tree = tree.split_off(Tree::block_capacity - 1); + expected.erase(expected.begin(), + expected.begin() + Tree::block_capacity - 1); + } else { + const auto keep = tree.size() - Tree::block_capacity + 1; + auto discarded = tree.split_off(keep); + expected.resize(keep); + } + std::vector sizes; + tree.for_each_block( + [&](const auto& block) { sizes.push_back(block.size()); }); + ASSERT_EQ(first ? sizes.front() : sizes.back(), 1); + const auto boundaries = tree.test_child_boundaries().front(); + ASSERT_GE(boundaries.size(), 4); + const auto left = whole || first ? 0 : boundaries[1]; + const auto right = + whole || !first ? tree.size() : boundaries[boundaries.size() - 2]; + const auto distance = boundaries[whole || first ? 1 : 2] - left; + const auto leaves = tree.test_leaf_identities(); + const auto nodes = tree.test_internal_node_identities(); + Tree::test_reset_counters(); + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(0); + EXPECT_THROW(tree.rotate_left(left, right, distance), std::bad_alloc); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(Tree::test_counters.child_transfers, 0); + EXPECT_EQ(Tree::test_counters.live_bytes, bytes); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + check(tree, expected); + tree.rotate_left(left, right, distance); + rotate(expected, left, right, distance); + check(tree, expected); + } + } +} + +TYPED_TEST(SequenceTreeRecyclingSpec, SingleLeafRotationUsesOneDescent) { + using Tree = TypeParam; + for (const std::size_t leaf_count : {1, 512}) { + auto expected = data(leaf_count * Tree::block_capacity); + auto tree = build(expected); + const auto leaves = tree.test_leaf_identities(); + const auto nodes = tree.test_internal_node_identities(); + const auto start = (leaf_count / 2) * Tree::block_capacity; + for (std::size_t l = 0; l < Tree::block_capacity; ++l) { + for (auto r = l + 2; r <= Tree::block_capacity; ++r) { + Tree::test_reset_counters(); + Tree::test_fail_after(0); + EXPECT_NO_THROW(tree.rotate_left(start + l, start + r, 1)); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.node_visits, tree.height()); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_EQ(Tree::test_counters.payload_mutations, 1); + EXPECT_EQ(Tree::test_counters.child_transfers, 0); + rotate(expected, start + l, start + r, 1); + check(tree, expected); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + } + } + } +} + +TYPED_TEST(SequenceTreeRecyclingSpec, SplitPrefixBudgetAcrossFragmentedShapes) { + using Tree = TypeParam; + /** + * @brief Exhaust cuts in full and repeatedly fragmented trees. + * @details Force empty, singleton and grouped siblings at different heights, + * including F4's minimally occupied ancestors. Sanitizer builds assert pool + * availability at every pop; fail-after forbids extra commit allocation. + */ + for (const std::size_t n : {63, 127, 257, 1025}) { + for (const bool fragmented : {false, true}) { + auto expected = data(n); + if (fragmented) { + for (std::size_t i = 1; i <= 3; ++i) { + rotate(expected, i, n - i, n / (i + 2)); + } + } + for (std::size_t p = 0; p <= n; ++p) { + auto tree = build(data(n)); + if (fragmented) { + for (std::size_t i = 1; i <= 3; ++i) { + tree.rotate_left(i, n - i, n / (i + 2)); + } + } + Tree::test_reset_counters(); + const auto budget = 3 * tree.height() + 1; + Tree::test_fail_after(budget); + Tree right; + EXPECT_NO_THROW(right = tree.split_off(p)); + Tree::test_fail_after(-1); + EXPECT_LE(Tree::test_counters.allocations, budget); + check(tree, {expected.begin(), expected.begin() + p}); + check(right, {expected.begin() + p, expected.end()}); + } + } + } +} + +TYPED_TEST(SequenceTreeRecyclingSpec, GroupedSeamWorkAndOffPathIdentity) { + using Tree = TypeParam; + constexpr auto capacity = Tree::block_capacity; + const std::array, 4> shapes{ + {{4096, 4096}, {4096, 1}, {1, 4096}, {4096, 4096}}}; + for (std::size_t shape = 0; shape < shapes.size(); ++shape) { + SCOPED_TRACE(shape); + auto left_values = data(shapes[shape][0] * capacity, 37); + auto right_values = data(shapes[shape][1] * capacity, 41); + auto tree = build(left_values); + auto donor = build(right_values); + { + const auto keep = + tree.size() - capacity + (shape == 3 ? capacity / 2 - 1 : 1); + auto discarded = tree.split_off(keep); + left_values.resize(keep); + if (shape != 3) { + auto rest = donor.split_off(capacity - 1); + donor = std::move(rest); + right_values.erase(right_values.begin(), + right_values.begin() + capacity - 1); + } + } + // Shapes 0..2 compact to three/two leaves; shape 3 retains all four. + // A seam root below F/2 occupancy must not escape as a nonroot node. + const auto h = std::max(tree.height(), donor.height()); + const auto budget = 2 * h + 4; + auto leaves = tree.test_leaf_identities(); + const auto donor_leaves = donor.test_leaf_identities(); + leaves.insert(leaves.end(), donor_leaves.begin(), donor_leaves.end()); + auto nodes = tree.test_internal_node_identities(); + const auto donor_nodes = donor.test_internal_node_identities(); + nodes.insert(nodes.end(), donor_nodes.begin(), donor_nodes.end()); + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(budget); + tree.merge(donor); + Tree::test_fail_after(-1); + const auto counters = Tree::test_counters; + EXPECT_EQ(counters.allocations, budget); + EXPECT_EQ(counters.peak_bytes - bytes, budget * Tree::node_storage_bytes); + // Exact work for these full-spine fixtures, not an empirical reserve bound. + // Reintroducing one join per seam leaf adds repeated taller-spine visits. + if (shape == 0) { + EXPECT_EQ(counters.node_visits, 23 * h - 6); + } else if (shape == 1) { + EXPECT_EQ(counters.node_visits, 13 * h - 3); + } else if (shape == 2) { + EXPECT_EQ(counters.node_visits, 11 * h - 5); + } + EXPECT_LE(counters.node_visits, 24 * h); + EXPECT_LE(counters.payload_mutations, 4); + EXPECT_LE(IntegerBlock::copied_elements, 8 * capacity); + EXPECT_GE(common_identity_count(leaves, tree.test_leaf_identities()) + 4, + leaves.size()); + EXPECT_GE( + common_identity_count(nodes, tree.test_internal_node_identities()) + + counters.node_visits, + nodes.size()); + left_values.insert(left_values.end(), right_values.begin(), + right_values.end()); + check(tree, left_values); + check(donor, {}); + } + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TYPED_TEST(SequenceTreeRecyclingSpec, DeepFragmentedRandomOwnership) { + using Tree = TypeParam; + auto expected = data(4096 * Tree::block_capacity + 3); + auto tree = build(expected); + std::mt19937_64 random(716381); + for (std::size_t step = 0; step < 80; ++step) { + SCOPED_TRACE(step); + const auto p = random() % (tree.size() + 1); + auto tail = tree.split_off(p); + check(tree, {expected.begin(), expected.begin() + p}); + check(tail, {expected.begin() + p, expected.end()}); + // Reverse the pieces and repeatedly relocate underfull exterior leaves. + tail.merge(tree); + EXPECT_TRUE(tree.empty()); + tree = std::move(tail); + rotate(expected, 0, expected.size(), p); + const auto left = random() % tree.size(); + const auto right = left + random() % (tree.size() - left + 1); + const auto distance = random(); + tree.rotate_left(left, right, distance); + rotate(expected, left, right, distance); + check(tree, expected); + const auto memory = tree.memory_usage(); + EXPECT_EQ(Tree::test_counters.live_allocations, + memory.blocks + memory.nodes); + EXPECT_EQ(Tree::test_counters.live_bytes, + memory.block_bytes + memory.node_bytes); + } +} + +template +void locality() { + using Tree = SequenceTree; + for (const std::size_t leaves : {64, 1024, 16384}) { + auto expected = data(leaves * IntegerBlock::capacity); + auto tree = build(expected); + const auto identities = tree.test_leaf_identities(); + const auto original_nodes = tree.test_internal_node_identities(); + EXPECT_EQ(original_nodes.size(), tree.node_count()); + const auto h = tree.height(); + auto bound = [&](std::size_t mutations) { + EXPECT_LE(Tree::test_counters.node_visits, 160 * (h + 4)); + EXPECT_LE(Tree::test_counters.child_transfers, 160 * 4 * (h + 4)); + EXPECT_LE(Tree::test_counters.payload_mutations, mutations); + EXPECT_LE(IntegerBlock::copied_elements, + mutations * 2 * IntegerBlock::capacity); + }; + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + auto tail = tree.split_off(tree.size() / 2 + 1); + bound(1); + EXPECT_LE(Tree::test_counters.allocations, 3 * h + 1); + EXPECT_TRUE(tree.test_validate()); + EXPECT_TRUE(tail.test_validate()); + auto split_nodes = tree.test_internal_node_identities(); + const auto tail_nodes = tail.test_internal_node_identities(); + split_nodes.insert(split_nodes.end(), tail_nodes.begin(), tail_nodes.end()); + EXPECT_EQ(split_nodes.size(), tree.node_count() + tail.node_count()); + const auto split_survivors = + common_identity_count(original_nodes, split_nodes); + EXPECT_GE(split_survivors + 32 * (h + 1), original_nodes.size()); + if (leaves == 16384) { + EXPECT_GT(split_survivors, original_nodes.size() / 2); + } + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + tree.merge(tail); + bound(5); + EXPECT_LE(Tree::test_counters.allocations, 2 * h + 4); + check(tree, expected); + const auto before_rotation_nodes = tree.test_internal_node_identities(); + Tree::test_reset_counters(); + IntegerBlock::copied_elements = 0; + const auto rotation_height = tree.height(); + tree.rotate_left(1, tree.size() - 1, tree.size() / 3); + bound(18); + EXPECT_LE(Tree::test_counters.allocations, 15 * rotation_height + 24 + 3); + rotate(expected, 1, expected.size() - 1, expected.size() / 3); + check(tree, expected); + const auto after_rotation_nodes = tree.test_internal_node_identities(); + EXPECT_EQ(after_rotation_nodes.size(), tree.node_count()); + const auto rotation_survivors = + common_identity_count(before_rotation_nodes, after_rotation_nodes); + EXPECT_GE(rotation_survivors + 128 * (rotation_height + 4), + before_rotation_nodes.size()); + if (leaves == 16384) { + EXPECT_GT(rotation_survivors, before_rotation_nodes.size() / 2); + } + EXPECT_GE(common_identity_count(identities, tree.test_leaf_identities()), + leaves - 16); + } +} +TEST(SequenceTreeLocality, CumulativeSpinesNotLeafScans) { + locality(); +} +TEST(SequenceTreeLocality, IndividualSpinesNotLeafScans) { + locality(); +} + +// Uniform conceptual elements allow full-width counts without huge allocations. +struct WeightedBlock { + using value_type = std::uint32_t; + static constexpr std::size_t capacity = + (std::numeric_limits::max() / 4) & ~std::size_t{1}; + std::size_t n = 0; + std::size_t size() const noexcept { return n; } + value_type operator[](std::size_t i) const { + assert(i < n); + (void)i; + return 7; + } + void rotate_left(std::size_t l, std::size_t r, std::size_t) noexcept { + assert(l <= r && r <= n); + (void)l; + (void)r; + } + void redistribute(WeightedBlock& rhs, std::size_t left) noexcept { + assert(this != &rhs && left <= capacity && left <= n + rhs.n && + n + rhs.n - left <= capacity); + rhs.n = n + rhs.n - left; + n = left; + } +}; +template +void full_width_counts() { + using Tree = SequenceTree; + constexpr auto maximum = std::numeric_limits::max(); + { + std::array full{{{WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {WeightedBlock::capacity - 1}}}; + auto aligned = Tree::from_blocks(full); + const auto ids = aligned.test_leaf_identities(); + Tree::test_reset_counters(); + Tree::test_fail_after(0); + EXPECT_NO_THROW( + aligned.rotate_left(0, aligned.size(), 2 * WeightedBlock::capacity)); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_TRUE(aligned.test_validate()); + auto expected = ids; + std::rotate(expected.begin(), expected.begin() + 2, expected.end()); + EXPECT_EQ(aligned.test_leaf_identities(), expected); + } + std::array blocks{ + {{WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {maximum - 4 * WeightedBlock::capacity}}}; + auto tree = Tree::from_blocks(blocks); + ASSERT_EQ(tree.size(), maximum); + ASSERT_TRUE(tree.test_validate()); + for (const auto p : {std::size_t{1}, WeightedBlock::capacity, maximum / 2, + maximum / 2 + 1, maximum - 1}) { + EXPECT_EQ(tree[p], 7); + auto suffix = tree.split_off(p); + EXPECT_EQ(tree.size(), p); + EXPECT_EQ(suffix.size(), maximum - p); + EXPECT_TRUE(tree.test_validate()); + EXPECT_TRUE(suffix.test_validate()); + tree.merge(suffix); + EXPECT_EQ(tree.size(), maximum); + EXPECT_TRUE(tree.test_validate()); + } + std::array extra{{{1}}}; + auto donor = Tree::from_blocks(extra); + Tree::test_fail_after(0); + EXPECT_THROW(tree.merge(donor), std::length_error); + EXPECT_THROW(tree.rotate_left(1, 0, 0), std::out_of_range); + Tree::test_fail_after(-1); + EXPECT_EQ(tree.size(), maximum); + EXPECT_EQ(donor.size(), 1); + tree.rotate_left(1, maximum - 1, maximum / 2); + EXPECT_TRUE(tree.test_validate()); + EXPECT_EQ(tree.size(), maximum); + std::array overflow{{{WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {WeightedBlock::capacity}, + {7}, + {1}}}; + EXPECT_THROW(Tree::from_blocks(overflow), std::length_error); +} +TEST(SequenceTreeCounts, CumulativeUnsignedHighBitAndOverflow) { + full_width_counts(); +} +TEST(SequenceTreeCounts, IndividualUnsignedHighBitAndOverflow) { + full_width_counts(); +} + +} // namespace diff --git a/src/tests/bits_unittests.cc b/src/tests/bits_unittests.cc index 8bc16ff..6522ba3 100644 --- a/src/tests/bits_unittests.cc +++ b/src/tests/bits_unittests.cc @@ -11,6 +11,80 @@ namespace { +TEST(PackedBitCopy, AllAlignmentsAndExactBackingExtents) { + pixie::copy_packed_bits(nullptr, 0, nullptr, 0, 0); + std::mt19937_64 random(91265); + for (size_t source_bit = 64; source_bit < 128; ++source_bit) { + for (size_t destination_bit = 64; destination_bit < 128; + ++destination_bit) { + for (const size_t count : + {0, 1, 2, 63, 64, 65, 127, 128, 129, + 255, 256, 257, 511, 512, 513, 1023, 1024, 1025, + 2047, 2048, 2049, 2111, 4095, 4096, 4097, 4159}) { + SCOPED_TRACE(::testing::Message() << source_bit << " -> " + << destination_bit << ": " << count); + // Exact extents catch SIMD lookahead past the last intersecting word. + std::vector source((source_bit + count + 63) / 64); + std::vector actual((destination_bit + count + 63) / 64); + for (auto& word : source) { + word = random(); + } + for (auto& word : actual) { + word = random(); + } + const auto original_source = source; + auto expected = actual; + for (size_t i = 0; i < count; ++i) { + const auto bit = + (source[(source_bit + i) / 64] >> ((source_bit + i) % 64)) & 1; + const auto p = destination_bit + i; + const auto mask = uint64_t{1} << (p % 64); + expected[p / 64] = (expected[p / 64] & ~mask) | (bit << (p % 64)); + } + pixie::copy_packed_bits(source.data(), source_bit, actual.data(), + destination_bit, count); + ASSERT_EQ(actual, expected); + ASSERT_EQ(source, original_source); + } + } + } +} + +TEST(PackedBitCopy, VectorWordAlignments) { + alignas(64) std::array source; + alignas(64) std::array actual; + std::mt19937_64 random(67234); + for (auto& word : source) { + word = random(); + } + for (size_t source_word = 0; source_word < 8; ++source_word) { + for (size_t destination_word = 0; destination_word < 8; + ++destination_word) { + for (size_t shift = 0; shift < 64; ++shift) { + for (const size_t count : {512, 1024, 2049}) { + SCOPED_TRACE(::testing::Message() + << source_word << " -> " << destination_word + << ": shift=" << shift << " count=" << count); + actual.fill(0xA5A5A5A5A5A5A5A5ull); + auto expected = actual; + const auto source_bit = source_word * 64 + shift; + const auto destination_bit = destination_word * 64; + for (size_t i = 0; i < count; ++i) { + const auto bit = + (source[(source_bit + i) / 64] >> ((source_bit + i) % 64)) & 1; + const auto p = destination_bit + i; + const auto mask = uint64_t{1} << (p % 64); + expected[p / 64] = (expected[p / 64] & ~mask) | (bit << (p % 64)); + } + pixie::copy_packed_bits(source.data(), source_bit, actual.data(), + destination_bit, count); + ASSERT_EQ(actual, expected); + } + } + } + } +} + using SelectBlock = std::array; uint64_t naive_select_512(const uint64_t* bits, uint64_t rank, bool value) { diff --git a/src/tests/packed_sequence_tests.cpp b/src/tests/packed_sequence_tests.cpp new file mode 100644 index 0000000..70c9b5e --- /dev/null +++ b/src/tests/packed_sequence_tests.cpp @@ -0,0 +1,413 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pixie; +using namespace pixie::detail::sequence; + +template +[[gnu::noinline]] void rotate_values(std::vector& values, + std::size_t left, + std::size_t right, + std::size_t distance) { + if (left != right) { + std::rotate(values.begin() + left, + values.begin() + left + distance % (right - left), + values.begin() + right); + } +} + +// Keep GCC 13's loop specialization from diagnosing impossible huge memcpy +// lengths in the exhaustive short-range tests (also affects std::rotate). +template +[[gnu::noinline]] void rotate_block(Block& block, + std::size_t left, + std::size_t right, + std::size_t distance) { + block.rotate_left(left, right, distance); +} + +template +auto contents(const Container& container) { + std::vector result; + for (std::size_t i = 0; i < container.size(); ++i) { + result.push_back(container[i]); + } + return result; +} + +TEST(PackedFieldRead, EveryShortFieldOriginWordAndRingCrossing) { + std::mt19937_64 random(271828); + for (const auto n : {0u, 1u, 3u, 63u, 64u, 65u, 127u, 128u, 129u, 192u}) { + std::array words{random(), random(), random()}; + if (n % 64) { + words[n / 64] |= ~std::uint64_t{0} << (n % 64); + } + BitBlock<192> block(words, n); + EXPECT_EQ(block.read_bits(n, 0), 0); + for (std::size_t origin = 0; origin < n; ++origin) { + for (std::size_t start = 0; start <= n; ++start) { + std::uint64_t expected = 0; + for (std::size_t width = 0; + width <= std::min(64, n - start); ++width) { + if (width != 0) { + const auto bit = (origin + start + width - 1) % n; + expected |= ((words[bit / 64] >> (bit % 64)) & 1) << (width - 1); + } + ASSERT_EQ(block.read_bits(start, width), expected) + << "n=" << n << " origin=" << origin << " start=" << start + << " width=" << width; + } + } + block.rotate_left(0, n, 1); + } + } +} + +template +class PackedValueSpec : public ::testing::Test {}; +using ValueBlocks = ::testing::Types, + PackedValueBlock, + PackedValueBlock, + PackedValueBlock, + PackedValueBlock, + PackedValueBlock, + PackedValueBlock, + PackedValueBlock>; +TYPED_TEST_SUITE(PackedValueSpec, ValueBlocks); + +TYPED_TEST(PackedValueSpec, LayoutBoundsEveryShortRotationAndWrappedReads) { + using Block = TypeParam; + using T = typename Block::value_type; + static_assert(SequenceBlock); + static_assert(sizeof(Block) == 64); + static_assert(alignof(Block) == 64); + constexpr auto maximum = ~std::uint64_t{0} >> (64 - Block::width); + std::mt19937_64 random(42); + std::array fields{}; + for (auto& field : fields) { + field = static_cast(random() & maximum); + } + fields.back() = static_cast(maximum); + Block empty; + EXPECT_TRUE(empty.empty()); + EXPECT_THROW(empty[0], std::out_of_range); + EXPECT_EQ(empty.payload_capacity_bytes() + empty.metadata_bytes(), + sizeof(Block)); + for (std::size_t n = 0; n <= std::min(8, Block::capacity); ++n) { + for (std::size_t left = 0; left <= n; ++left) { + for (std::size_t right = left; right <= n; ++right) { + for (std::size_t d = 0; d <= right - left + 1; ++d) { + Block block(std::span(fields.data(), n)); + std::vector expected(fields.begin(), fields.begin() + n); + rotate_block(block, left, right, d); + rotate_values(expected, left, right, d); + EXPECT_EQ(contents(block), expected); + } + } + } + } + Block block{std::span(fields)}; + std::vector expected(fields.begin(), fields.end()); + for (std::size_t i = 0; i < Block::capacity; ++i) { + block.rotate_left(0, block.size(), 1); + rotate_values(expected, 0, expected.size(), 1); + EXPECT_EQ(contents(block), expected); + auto partial = block; + auto oracle = expected; + partial.rotate_left(1, partial.size() - 1, + std::numeric_limits::max()); + rotate_values(oracle, 1, oracle.size() - 1, + std::numeric_limits::max()); + EXPECT_EQ(contents(partial), oracle); + } + EXPECT_THROW(block[block.size()], std::out_of_range); + std::array oversized{}; + EXPECT_THROW((Block{std::span(oversized)}), std::invalid_argument); + if constexpr (Block::width < std::numeric_limits::digits) { + const std::array invalid{static_cast(maximum + 1)}; + EXPECT_THROW((Block{std::span(invalid)}), std::invalid_argument); + } +} + +TYPED_TEST(PackedValueSpec, WrappedRedistributionAndBiasReencoding) { + using Block = TypeParam; + using T = typename Block::value_type; + constexpr auto maximum = ~std::uint64_t{0} >> (64 - Block::width); + std::array fields{}; + for (std::size_t i = 0; i < fields.size(); ++i) { + fields[i] = static_cast(i & maximum); + } + Block source{std::span(fields)}; + source.rotate_left(0, source.size(), 3); + const auto original = contents(source); + for (std::size_t cut = 0; cut <= source.size(); ++cut) { + auto left = source; + Block right; + left.redistribute(right, cut); + EXPECT_EQ(contents(left), + (std::vector(original.begin(), original.begin() + cut))); + EXPECT_EQ(contents(right), + (std::vector(original.begin() + cut, original.end()))); + left.redistribute(right, source.size()); + EXPECT_TRUE(right.empty()); + EXPECT_EQ(contents(left), original); + } + const auto half = Block::capacity / 2; + Block a(std::span(fields.data(), half)); + Block b(std::span(fields.data(), half + 1)); + a.rotate_left(0, a.size(), 1); + b.rotate_left(0, b.size(), 2); + auto expected = contents(a); + const auto second = contents(b); + expected.insert(expected.end(), second.begin(), second.end()); + a.redistribute(b, Block::capacity); + auto joined = contents(a); + const auto tail = contents(b); + joined.insert(joined.end(), tail.begin(), tail.end()); + EXPECT_EQ(joined, expected); + + fields.fill(T{0}); + Block biased{std::span(fields)}; + biased.rotate_left(0, biased.size(), 3); + biased.add_bias(maximum); + for (std::size_t i = 0; i < biased.size(); ++i) { + EXPECT_EQ(biased[i], static_cast(maximum)); + } + biased.add_bias(0); + EXPECT_EQ(biased[0], static_cast(maximum)); +} + +template +class TaggedTreeSpec : public ::testing::Test {}; +using TaggedTrees = + ::testing::Types, + 4, + LengthLayout::cumulative, + true>, + SequenceTree, + 4, + LengthLayout::individual, + true>, + SequenceTree, + 8, + LengthLayout::cumulative, + true>, + SequenceTree, + 8, + LengthLayout::individual, + true>, + SequenceTree, + 16, + LengthLayout::cumulative, + true>, + SequenceTree, + 16, + LengthLayout::individual, + true>>; +TYPED_TEST_SUITE(TaggedTreeSpec, TaggedTrees); + +template +Tree make_tagged(std::size_t n) { + using Block = typename Tree::block_type; + auto blocks = + std::views::iota(std::size_t{0}, + n / Block::capacity + (n % Block::capacity != 0)) | + std::views::transform([n](std::size_t index) { + std::array fields; + const auto start = index * Block::capacity; + const auto count = std::min(Block::capacity, n - start); + std::iota(fields.begin(), fields.end(), start); + return Block(std::span(fields.data(), count)); + }); + return Tree::from_blocks(blocks); +} + +TYPED_TEST(TaggedTreeSpec, FullWidthBiasConstReadsNestedSplitsAndRejoins) { + using Tree = TypeParam; + constexpr auto maximum = std::numeric_limits::max(); + constexpr std::size_t n = Tree::block_capacity * 103 + 1; + auto tree = make_tagged(n); + tree.test_add_bias(std::uint64_t{1} << 63); + tree.test_add_bias(maximum - (std::uint64_t{1} << 63) - (n - 1)); + std::vector expected(n); + std::iota(expected.begin(), expected.end(), maximum - (n - 1)); + const auto leaves = tree.test_leaf_identities(); + const auto nodes = tree.test_internal_node_identities(); + const auto biases = tree.test_bias_snapshot(); + Tree::test_reset_counters(); + EXPECT_EQ(contents(std::as_const(tree)), expected); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + EXPECT_EQ(tree.test_bias_snapshot(), biases); + ASSERT_TRUE(tree.test_validate()); + std::mt19937_64 random(173); + for (std::size_t iteration = 0; iteration < 70; ++iteration) { + const auto cut = + iteration < Tree::block_capacity ? iteration : random() % n; + const auto h = tree.height(); + Tree::test_reset_counters(); + auto right = tree.split_off(cut); + EXPECT_LE(Tree::test_counters.allocations, 3 * h + 1); + EXPECT_LE(Tree::test_counters.payload_mutations, 3); + ASSERT_TRUE(tree.test_validate()); + ASSERT_TRUE(right.test_validate()); + EXPECT_EQ(contents(tree), (std::vector( + expected.begin(), expected.begin() + cut))); + EXPECT_EQ(contents(right), (std::vector( + expected.begin() + cut, expected.end()))); + tree.merge(right); + EXPECT_TRUE(right.empty()); + const auto left = random() % n; + const auto end = left + random() % (n - left + 1); + const auto distance = random(); + tree.rotate_left(left, end, distance); + rotate_values(expected, left, end, distance); + ASSERT_TRUE(tree.test_validate()); + EXPECT_EQ(contents(tree), expected); + } +} + +TYPED_TEST(TaggedTreeSpec, ChildRotationRetainsUniformParentBias) { + using Tree = TypeParam; + auto tree = make_tagged(257 * Tree::block_capacity); + tree.test_add_bias(std::uint64_t{1} << 63); + const auto raw = tree.test_bias_snapshot(); + const auto boundaries = tree.test_child_boundaries().front(); + auto expected = contents(tree); + Tree::test_reset_counters(); + Tree::test_fail_after(0); + EXPECT_NO_THROW(tree.rotate_left(0, tree.size(), boundaries[1])); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + const auto after = tree.test_bias_snapshot(); + ASSERT_EQ(after.size(), raw.size()); + EXPECT_EQ(after.front(), raw.front()); + for (const auto& entry : after) { + EXPECT_NE(std::ranges::find(raw, entry), raw.end()); + } + rotate_values(expected, 0, expected.size(), boundaries[1]); + EXPECT_EQ(contents(tree), expected); + EXPECT_TRUE(tree.test_validate()); +} + +TYPED_TEST(TaggedTreeSpec, TaggedLeafFittingMergeAndOffPathOwnership) { + using Tree = TypeParam; + auto a = make_tagged(2); + auto b = make_tagged(3); + a.test_add_bias(17); + b.test_add_bias(std::numeric_limits::max() - 2); + auto expected = contents(a); + const auto donor = contents(b); + expected.insert(expected.end(), donor.begin(), donor.end()); + Tree::test_fail_after(0); + a.rotate_left(0, 2, 1); + a.merge(b); + Tree::test_fail_after(-1); + std::swap(expected[0], expected[1]); + EXPECT_EQ(contents(a), expected); + ASSERT_TRUE(a.test_validate()); + auto deep = make_tagged(Tree::block_capacity * 1000); + deep.test_add_bias(12345); + const auto leaves = deep.test_leaf_identities(); + const auto nodes = deep.test_internal_node_identities(); + const auto h = deep.height(); + Tree::test_reset_counters(); + deep.rotate_left(1, deep.size() - 1, deep.size() / 3); + const auto counts = Tree::test_counters; + EXPECT_EQ(counts.allocations, 15 * h + 27); + EXPECT_LE(counts.node_visits, 160 * (h + 4)); + EXPECT_LE(counts.payload_mutations, 45); + const auto after = deep.test_leaf_identities(); + std::size_t surviving = 0; + for (auto p : leaves) { + surviving += std::find(after.begin(), after.end(), p) != after.end(); + } + EXPECT_GE(surviving + 12, leaves.size()); + const auto after_nodes = deep.test_internal_node_identities(); + surviving = 0; + for (auto p : nodes) { + surviving += std::find(after_nodes.begin(), after_nodes.end(), p) != + after_nodes.end(); + } + EXPECT_GE(surviving + counts.node_visits, nodes.size()); + ASSERT_TRUE(deep.test_validate()); +} + +TYPED_TEST(TaggedTreeSpec, SplitFailureEveryPointAndMultipleCutsInsideOneLeaf) { + using Tree = TypeParam; + constexpr auto capacity = Tree::block_capacity; + constexpr auto n = capacity * 43 + 1; + auto make = [] { + auto tree = make_tagged(n); + tree.test_add_bias(std::uint64_t{1} << 62); + tree.rotate_left(0, n, 1); + tree.test_add_bias(std::uint64_t{1} << 62); + return tree; + }; + std::size_t allocations; + { + auto tree = make(); + Tree::test_reset_counters(); + auto right = tree.split_off(capacity + 1); + allocations = Tree::test_counters.allocations; + } + for (std::size_t fail = 0; fail <= allocations; ++fail) { + auto tree = make(); + const auto expected = contents(tree); + const auto leaves = tree.test_leaf_identities(); + const auto nodes = tree.test_internal_node_identities(); + const auto biases = tree.test_bias_snapshot(); + Tree right; + const auto right_biases = right.test_bias_snapshot(); + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_reset_counters(); + Tree::test_fail_after(fail); + if (fail < allocations) { + EXPECT_THROW(right = tree.split_off(capacity + 1), std::bad_alloc); + } else { + EXPECT_NO_THROW(right = tree.split_off(capacity + 1)); + } + Tree::test_fail_after(-1); + if (fail < allocations) { + EXPECT_EQ(Tree::test_counters.live_bytes, bytes); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(tree.test_leaf_identities(), leaves); + EXPECT_EQ(tree.test_internal_node_identities(), nodes); + EXPECT_EQ(tree.test_bias_snapshot(), biases); + EXPECT_EQ(right.test_bias_snapshot(), right_biases); + } else { + tree.merge(right); + } + EXPECT_EQ(contents(tree), expected); + ASSERT_TRUE(tree.test_validate()); + } + for (auto distance : {std::size_t{1}, capacity - 2}) { + auto tree = make(); + auto expected = contents(tree); + tree.rotate_left(1, n - 1, distance); + rotate_values(expected, 1, n - 1, distance); + ASSERT_TRUE(tree.test_validate()); + EXPECT_EQ(contents(tree), expected); + } +} + +} // namespace diff --git a/src/tests/permutable_sequence_tests.cpp b/src/tests/permutable_sequence_tests.cpp new file mode 100644 index 0000000..a28d9e3 --- /dev/null +++ b/src/tests/permutable_sequence_tests.cpp @@ -0,0 +1,930 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pixie; +using namespace pixie::detail::sequence; + +template +struct ResetFailures { + ~ResetFailures() { + Sequence::test_fail_payload_after(-1); + Sequence::test_tree_type::test_fail_after(-1); + } +}; + +template +std::vector values(std::size_t size, std::uint64_t seed = 42) { + std::mt19937_64 random(seed); + std::vector result; + result.reserve(size); + for (std::size_t i = 0; i < size; ++i) { + if constexpr (std::same_as) { + result.push_back(std::to_string(random()) + std::string(40, 'x')); + } else { + result.push_back(static_cast(random())); + } + } + return result; +} + +template +void rotate(std::vector& input, + std::size_t left, + std::size_t right, + std::size_t distance) { + if (left != right) { + std::rotate(input.begin() + left, + input.begin() + left + distance % (right - left), + input.begin() + right); + } +} + +template +void check(const Sequence& sequence, + const std::vector& expected) { + ASSERT_TRUE(sequence.test_tree().test_validate()); + ASSERT_EQ(sequence.size(), expected.size()); + ASSERT_EQ(sequence.empty(), expected.empty()); + for (std::size_t i = 0; i < expected.size(); ++i) { + ASSERT_EQ(sequence[i], expected[i]) << "index=" << i; + } + const auto memory = sequence.memory_usage(); + EXPECT_EQ(memory.facade_bytes, sizeof(sequence)); + EXPECT_EQ(memory.total_bytes, sizeof(sequence) + memory.tree_block_bytes + + memory.tree_node_bytes + + memory.chunk_header_bytes + + memory.vector_capacity_bytes); + EXPECT_EQ(memory.vector_capacity_bytes, + memory.vector_live_bytes + memory.vector_slack_bytes); + EXPECT_EQ(sequence.memory_usage_bytes(), memory.total_bytes); + if constexpr (Sequence::storage == ElementStorage::packed) { + EXPECT_EQ(memory.order_bytes, 0); + EXPECT_EQ(memory.chunks, 0); + EXPECT_EQ(memory.vector_capacity_bytes, 0); + EXPECT_GE(memory.packed_capacity_bits, + sequence.size() * + std::numeric_limits::digits); + } else { + EXPECT_EQ(memory.packed_capacity_bits, 0); + EXPECT_EQ(memory.order_bytes, + memory.tree_block_bytes + memory.tree_node_bytes); + EXPECT_EQ(memory.vector_live_bytes, + sequence.size() * sizeof(typename Sequence::value_type)); + } + if (sequence.empty()) { + EXPECT_EQ(memory.total_bytes, sizeof(sequence)); + EXPECT_EQ(memory.chunks, 0); + } +} + +template +class PermutableSequenceSpec : public ::testing::Test {}; +using Sequences = ::testing::Types< + PermutableSequence, + PermutableSequence, + PermutableSequence, + PermutableSequence, + PermutableSequence, + PermutableSequence, + PermutableSequence, + PermutableSequence, + PermutableSequence>; +TYPED_TEST_SUITE(PermutableSequenceSpec, Sequences); + +template +concept CanConsume = + requires(Range&& range) { S::from_range(std::forward(range)); }; + +TYPED_TEST(PermutableSequenceSpec, PublicContractAliasesNoexceptAndLayout) { + using S = TypeParam; + using T = typename S::value_type; + using R = typename S::const_reference; + using Base = PermutableSequenceBase; + static_assert(std::is_base_of_v && std::is_empty_v); + static_assert(std::same_as); + static_assert(std::same_as()[0]), R>); + static_assert(noexcept(std::declval().size())); + static_assert(noexcept(std::declval().empty())); + static_assert(noexcept(std::declval().memory_usage_bytes())); + static_assert(!noexcept(std::declval()[0])); + static_assert(!CanConsume); + static_assert(!CanConsume, 2>&>); + static_assert( + sizeof(S) == + sizeof(typename S::test_tree_type) + + (S::storage == ElementStorage::packed ? 0 : 2 * sizeof(void*))); + auto input = values(3); + static_assert(std::same_as); + auto owner = Base::from_range(input); + Base& contract = owner; + EXPECT_EQ(contract.size(), 3); + EXPECT_THROW(contract[3], std::out_of_range); + EXPECT_THROW(contract.rotate_left(0, 4, 0), std::out_of_range); + EXPECT_EQ(contract.memory_usage_bytes(), owner.memory_usage().total_bytes); + if constexpr (std::is_reference_v) { + const T* first = &contract[0]; + S receiver; + receiver.merge(owner); + EXPECT_EQ(&receiver[0], first); + } +} + +TYPED_TEST(PermutableSequenceSpec, EmptyCheckedRangesAndExclusiveMoves) { + using Sequence = TypeParam; + using T = typename Sequence::value_type; + static_assert(!std::is_copy_constructible_v); + static_assert(!std::is_copy_assignable_v); + static_assert(std::is_nothrow_move_constructible_v); + static_assert(std::is_nothrow_move_assignable_v); + if constexpr (Sequence::storage == ElementStorage::packed) { + static_assert( + std::same_as()[0]), T>); + } else { + static_assert( + std::same_as()[0]), const T&>); + } + Sequence sequence; + check(sequence, {}); + EXPECT_THROW(sequence[0], std::out_of_range); + EXPECT_THROW(sequence.rotate_left(1, 0, 0), std::out_of_range); + EXPECT_THROW(sequence.rotate_left(0, 1, 0), std::out_of_range); + EXPECT_NO_THROW(sequence.rotate_left(0, 0, SIZE_MAX)); + auto input = values(117); + const auto expected = input; + sequence = Sequence::from_range(input); + check(sequence, expected); + EXPECT_THROW(sequence[sequence.size()], std::out_of_range); + EXPECT_THROW(sequence.rotate_left(10, 9, 0), std::out_of_range); + EXPECT_THROW(sequence.rotate_left(0, sequence.size() + 1, 0), + std::out_of_range); + sequence.merge(sequence); + auto* self = &sequence; + sequence = std::move(*self); + check(sequence, expected); + Sequence moved(std::move(sequence)); + check(sequence, {}); + sequence.merge(moved); + check(moved, {}); + check(sequence, expected); + sequence.merge(moved); + check(sequence, expected); + auto replacement_input = values(17, 71); + const auto replacement_expected = replacement_input; + moved = Sequence::from_range(replacement_input); + sequence = std::move(moved); + check(moved, {}); + check(sequence, replacement_expected); +} + +TYPED_TEST(PermutableSequenceSpec, DifferentialRotationsAndUnrebasedMerges) { + using Sequence = TypeParam; + using T = typename Sequence::value_type; + auto input = values(2309); + auto expected = input; + auto sequence = Sequence::from_range(input); + std::mt19937_64 random(174); + for (std::size_t step = 0; step < 80; ++step) { + if (step % 13 == 0) { + auto addition = values(step + 1, step); + expected.insert(expected.end(), addition.begin(), addition.end()); + auto donor = Sequence::from_range(addition); + sequence.merge(donor); + check(donor, {}); + } + auto left = random() % (expected.size() + 1); + auto right = random() % (expected.size() + 1); + if (left > right) { + std::swap(left, right); + } + if (step % 7 == 0) { + left = 0; + right = expected.size(); + } + const auto distance = random(); + sequence.rotate_left(left, right, distance); + rotate(expected, left, right, distance); + check(sequence, expected); + } +} + +TYPED_TEST(PermutableSequenceSpec, EveryShortRotationAndBoundaryConstruction) { + using Sequence = TypeParam; + using T = typename Sequence::value_type; + for (std::size_t size = 0; size <= 7; ++size) { + const auto original = values(size); + for (std::size_t left = 0; left <= size; ++left) { + for (std::size_t right = left; right <= size; ++right) { + for (std::size_t distance = 0; distance <= size + 1; ++distance) { + auto input = original; + auto expected = original; + auto sequence = Sequence::from_range(input); + sequence.rotate_left(left, right, distance); + rotate(expected, left, right, distance); + check(sequence, expected); + } + } + } + } + constexpr auto capacity = Sequence::test_tree_type::block_capacity; + for (auto size : {capacity - 1, capacity, capacity + 1, 2 * capacity + 1}) { + auto input = values(size); + auto expected = input; + auto sequence = Sequence::from_range(input); + check(sequence, expected); + } +} + +TEST(PermutableSequence, CompileTimeSelectionAndExactPointerBudget) { + static_assert(PermutableSequence::storage == ElementStorage::packed); + static_assert(PermutableSequence::storage == + ElementStorage::packed); + static_assert(PermutableSequence::storage == ElementStorage::indirect); + static_assert(PermutableSequence::storage == + ElementStorage::indirect); + static_assert(PermutableSequence::storage == + ElementStorage::indirect); + static_assert(SequenceBlock>); + static_assert(sizeof(pixie::detail::sequence::PointerBlock) == 64); + static_assert(sizeof(pixie::detail::sequence::PointerBlock) == + 256); + static_assert(pixie::detail::sequence::PointerBlock::capacity == + (64 - 2 * sizeof(std::size_t)) / sizeof(const int*)); + EXPECT_EQ(sizeof(PermutableSequence), + sizeof(PermutableSequence::test_tree_type)); +} + +TEST(PermutableSequence, NativePointerBlockWrappedRedistribution) { + using Block = pixie::detail::sequence::PointerBlock; + constexpr auto capacity = Block::capacity; + std::array objects{}; + std::array pointers{}; + for (std::size_t i = 0; i < pointers.size(); ++i) { + pointers[i] = &objects[i]; + } + EXPECT_THROW( + (Block(std::span(pointers.data(), capacity + 1))), + std::length_error); + for (std::size_t a = 0; a <= capacity; ++a) { + for (std::size_t b = 0; b <= capacity; ++b) { + for (std::size_t split = 0; split <= a + b; ++split) { + if (split > capacity || a + b - split > capacity) { + continue; + } + Block left(std::span(pointers.data(), a)); + Block right(std::span(pointers.data() + a, b)); + std::vector expected(pointers.begin(), + pointers.begin() + a + b); + left.rotate_left(0, a, 5); + right.rotate_left(0, b, 3); + rotate(expected, 0, a, 5); + rotate(expected, a, a + b, 3); + if (a > 2) { + left.rotate_left(1, a, 1); + rotate(expected, 1, a, 1); + } + left.redistribute(right, split); + ASSERT_EQ(left.size(), split); + ASSERT_EQ(right.size(), a + b - split); + for (std::size_t i = 0; i < split; ++i) { + EXPECT_EQ(left[i], expected[i]); + } + for (std::size_t i = split; i < a + b; ++i) { + EXPECT_EQ(right[i - split], expected[i]); + } + } + } + } +} + +TEST(PermutableSequence, SinglePassStreamInput) { + std::istringstream stream("1 2 3 4 5 6 7 8 9"); + auto input = std::ranges::istream_view(stream); + auto sequence = PermutableSequence::from_range(input); + check(sequence, {1, 2, 3, 4, 5, 6, 7, 8, 9}); +} + +struct MovingInput { + unsigned length = 23; + unsigned position = 0; + unsigned moves = 0; + unsigned throw_increment = std::numeric_limits::max(); + struct Iterator { + using value_type = unsigned; + using difference_type = std::ptrdiff_t; + using iterator_concept = std::input_iterator_tag; + MovingInput* input; + unsigned operator*() const { return input->position; } + Iterator& operator++() { + if (input->position == input->throw_increment) { + throw std::runtime_error("input increment"); + } + ++input->position; + return *this; + } + void operator++(int) { ++*this; } + bool operator==(std::default_sentinel_t) const { + return input->position == input->length; + } + friend unsigned iter_move(const Iterator& it) { + ++it.input->moves; + return it.input->position + 100; + } + }; + Iterator begin() { return {this}; } + std::default_sentinel_t end() const { return {}; } +}; +static_assert(std::ranges::input_range); +static_assert(!std::ranges::forward_range); + +TEST(PermutableSequence, HonorsCustomIteratorMoveAndCleansUpThrowingIncrement) { + using Packed = PermutableSequence; + using Indirect = PermutableSequence; + auto exercise = []() { + { + MovingInput input; + auto sequence = Sequence::from_range(input); + ASSERT_EQ(sequence.size(), input.length); + EXPECT_EQ(input.moves, input.length); + for (unsigned i = 0; i < input.length; ++i) { + EXPECT_EQ(sequence[i], i + 100); + } + } + for (unsigned fail = 0; fail < 23; ++fail) { + MovingInput input; + input.throw_increment = fail; + EXPECT_THROW(Sequence::from_range(input), std::runtime_error); + EXPECT_EQ(input.moves, fail + 1); + EXPECT_EQ(Sequence::test_tree_type::test_counters.live_allocations, 0); + } + }; + exercise.template operator()(); + exercise.template operator()(); +} + +// No default construction, copying, or move assignment is available. Slot +// ownership therefore cannot silently depend on vector resizing or T swaps. +struct alignas(128) Tracked { + inline static int live = 0; + inline static int moves = 0; + int value; + explicit Tracked(int n) : value(n) { ++live; } + Tracked(const Tracked&) = delete; + Tracked& operator=(const Tracked&) = delete; + Tracked(Tracked&& other) noexcept : value(std::exchange(other.value, -1)) { + ++live; + ++moves; + } + Tracked& operator=(Tracked&&) = delete; + ~Tracked() noexcept { --live; } +}; +static_assert(!std::is_default_constructible_v); +static_assert(!std::is_move_assignable_v); +using Indirect = PermutableSequence; + +Indirect tracked_sequence(int begin, int end) { + auto input = std::views::iota(begin, end) | + std::views::transform([](int i) { return Tracked(i); }); + return Indirect::from_range(input); +} + +struct ThrowingTrackedInput { + int length = 41; + int position = 0; + int comparisons = 0; + int throw_increment = -1; + int throw_comparison = -1; + int live_at_failure = -1; + std::size_t tree_allocations_at_failure = 0; + + [[noreturn]] void fail() { + live_at_failure = Tracked::live; + tree_allocations_at_failure = + Indirect::test_tree_type::test_counters.live_allocations; + throw std::runtime_error("tracked input failure"); + } + struct Iterator { + using value_type = Tracked; + using difference_type = std::ptrdiff_t; + using iterator_concept = std::input_iterator_tag; + ThrowingTrackedInput* input; + Tracked operator*() const { return Tracked(input->position); } + Iterator& operator++() { + if (input->position == input->throw_increment) { + input->fail(); + } + ++input->position; + return *this; + } + void operator++(int) { ++*this; } + bool operator==(std::default_sentinel_t) const { + if (input->comparisons++ == input->throw_comparison) { + input->fail(); + } + return input->position == input->length; + } + }; + Iterator begin() { return {this}; } + std::default_sentinel_t end() const { return {}; } +}; +static_assert(std::ranges::input_range); +static_assert(!std::ranges::forward_range); + +TEST(PermutableSequence, + TrackedIncrementAndSentinelFailuresReclaimPartialChunks) { + using Tree = Indirect::test_tree_type; + ASSERT_EQ(Tracked::live, 0); + const auto initial_allocations = Tree::test_counters.live_allocations; + { + auto existing = tracked_sequence(100, 107); + const auto baseline_live = Tracked::live; + const auto baseline_allocations = Tree::test_counters.live_allocations; + const auto baseline_bytes = Tree::test_counters.live_bytes; + ThrowingTrackedInput complete; + { + auto sequence = Indirect::from_range(complete); + ASSERT_EQ(sequence.size(), complete.length); + for (int i = 0; i < complete.length; ++i) { + EXPECT_EQ(sequence[i].value, i); + } + } + auto check_cleanup = [&] { + EXPECT_EQ(Tracked::live, baseline_live); + EXPECT_EQ(Tree::test_counters.live_allocations, baseline_allocations); + EXPECT_EQ(Tree::test_counters.live_bytes, baseline_bytes); + for (std::size_t i = 0; i < existing.size(); ++i) { + EXPECT_EQ(existing[i].value, 100 + static_cast(i)); + } + }; + check_cleanup(); + bool increment_after_published_leaf = false; + for (int failure = 0; failure < complete.length; ++failure) { + SCOPED_TRACE(failure); + ThrowingTrackedInput input; + input.throw_increment = failure; + EXPECT_THROW(Indirect::from_range(input), std::runtime_error); + // ++ runs after emplacement and destruction of the iterator temporary, + // so this includes the newly constructed, still unpublished slot. + EXPECT_EQ(input.live_at_failure, baseline_live + failure + 1); + increment_after_published_leaf |= + input.tree_allocations_at_failure > baseline_allocations; + check_cleanup(); + } + EXPECT_TRUE(increment_after_published_leaf); + bool comparison_with_partial_first_chunk = false; + bool comparison_after_published_leaf = false; + for (int failure = 0; failure < complete.comparisons; ++failure) { + SCOPED_TRACE(failure); + ThrowingTrackedInput input; + input.throw_comparison = failure; + EXPECT_THROW(Indirect::from_range(input), std::runtime_error); + EXPECT_GE(input.live_at_failure, baseline_live); + comparison_with_partial_first_chunk |= + input.live_at_failure > baseline_live && + input.live_at_failure < baseline_live + 4; + comparison_after_published_leaf |= + input.tree_allocations_at_failure > baseline_allocations; + check_cleanup(); + } + EXPECT_TRUE(comparison_with_partial_first_chunk); + EXPECT_TRUE(comparison_after_published_leaf); + } + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Tree::test_counters.live_allocations, initial_allocations); +} + +TEST(PermutableSequence, ConsumesExistingMoveOnlyObjectsExactlyOnce) { + ASSERT_EQ(Tracked::live, 0); + { + std::vector input; + input.reserve(17); + for (int i = 0; i < 17; ++i) { + input.emplace_back(i); + } + Tracked::moves = 0; + auto sequence = Indirect::from_range(input); + EXPECT_EQ(Tracked::moves, 17); + EXPECT_EQ(Tracked::live, 34); + for (int i = 0; i < 17; ++i) { + EXPECT_EQ(input[i].value, -1); + EXPECT_EQ(sequence[i].value, i); + EXPECT_NE(&sequence[i], &input[i]); + } + } + EXPECT_EQ(Tracked::live, 0); +} + +template +auto addresses(const Sequence& sequence) { + std::vector result; + for (std::size_t i = 0; i < sequence.size(); ++i) { + result.push_back(std::addressof(sequence[i])); + } + return result; +} + +template +void check_addresses( + const Sequence& sequence, + const std::vector& expected) { + ASSERT_EQ(sequence.size(), expected.size()); + EXPECT_TRUE(sequence.test_tree().test_validate()); + for (std::size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(std::addressof(sequence[i]), expected[i]); + } +} + +TEST(PermutableSequence, NonemptyMoveAssignmentDestroysOnlyReplacedPayload) { + using Tree = Indirect::test_tree_type; + ASSERT_EQ(Tracked::live, 0); + const auto initial_allocations = Tree::test_counters.live_allocations; + { + auto receiver = tracked_sequence(0, 17); + auto source = tracked_sequence(100, 129); + source.rotate_left(0, source.size(), 7); + const auto incoming_addresses = addresses(source); + std::vector incoming_values; + for (auto* value : incoming_addresses) { + incoming_values.push_back(value->value); + } + const auto old_count = receiver.size(); + const auto old_memory = receiver.test_tree().memory_usage(); + const auto before_live = Tracked::live; + const auto before_allocations = Tree::test_counters.live_allocations; + Tracked::moves = 0; + ResetFailures reset; + Tree::test_fail_after(0); + Indirect::test_fail_payload_after(0); + EXPECT_EQ(&(receiver = std::move(source)), &receiver); + check_addresses(receiver, incoming_addresses); + for (std::size_t i = 0; i < incoming_values.size(); ++i) { + EXPECT_EQ(receiver[i].value, incoming_values[i]); + } + EXPECT_EQ(Tracked::moves, 0); + EXPECT_EQ(Tracked::live, before_live - static_cast(old_count)); + EXPECT_EQ(Tree::test_counters.live_allocations, + before_allocations - old_memory.blocks - old_memory.nodes); + EXPECT_TRUE(source.empty()); + EXPECT_EQ(source.size(), 0); + EXPECT_EQ(source.memory_usage_bytes(), sizeof(source)); + EXPECT_EQ(source.memory_usage().chunks, 0); + EXPECT_TRUE(source.test_tree().test_validate()); + } + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Tree::test_counters.live_allocations, initial_allocations); +} + +TEST(PermutableSequence, + StableAlignedMoveOnlyPayloadNeverMovesAfterConstruction) { + ASSERT_EQ(Tracked::live, 0); + { + auto sequence = tracked_sequence(0, 173); + auto donor = tracked_sequence(173, 254); + auto expected = addresses(sequence); + auto donor_addresses = addresses(donor); + Tracked::moves = 0; + for (auto* address : expected) { + EXPECT_EQ(reinterpret_cast(address) % 128, 0); + } + sequence.rotate_left(3, 171, 37); + rotate(expected, 3, 171, 37); + donor.rotate_left(0, donor.size(), 23); + rotate(donor_addresses, 0, donor_addresses.size(), 23); + sequence.merge(donor); + expected.insert(expected.end(), donor_addresses.begin(), + donor_addresses.end()); + EXPECT_TRUE(donor.empty()); + EXPECT_EQ(donor.memory_usage_bytes(), sizeof(donor)); + auto moved = std::move(sequence); + EXPECT_TRUE(sequence.empty()); + sequence = std::move(moved); + EXPECT_TRUE(moved.empty()); + check_addresses(sequence, expected); + EXPECT_EQ(Tracked::moves, 0); + EXPECT_EQ(Tracked::live, 254); + auto memory = sequence.memory_usage(); + EXPECT_EQ(memory.chunks, (173 + 3) / 4 + (81 + 3) / 4); + EXPECT_EQ(memory.vector_live_bytes, 254 * sizeof(Tracked)); + EXPECT_EQ(memory.vector_capacity_bytes, + memory.vector_live_bytes + memory.vector_slack_bytes); + sequence = std::move(donor); + EXPECT_TRUE(sequence.empty()); + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Tracked::moves, 0); + } + EXPECT_EQ(Tracked::live, 0); +} + +TEST(PermutableSequence, ForcedIndirectBoolHasRealStableObjectReferences) { + using Sequence = PermutableSequence; + std::array input{}; + for (std::size_t i = 0; i < input.size(); ++i) { + input[i] = i % 3 == 0; + } + auto sequence = Sequence::from_range(input); + auto donor = Sequence::from_range(input); + auto expected = addresses(sequence); + const auto donor_addresses = addresses(donor); + expected.insert(expected.end(), donor_addresses.begin(), + donor_addresses.end()); + Sequence::test_fail_payload_after(0); + ResetFailures reset; + sequence.merge(donor); + sequence.rotate_left(1, 45, 13); + rotate(expected, 1, 45, 13); + check_addresses(sequence, expected); + for (std::size_t i = 0; i < sequence.size(); ++i) { + const bool& reference = sequence[i]; + EXPECT_EQ(reference, *expected[i]); + for (std::size_t j = 0; j < i; ++j) { + EXPECT_NE(expected[i], expected[j]); + } + } + EXPECT_EQ(sequence.memory_usage().chunks, 8); +} + +TEST(PermutableSequence, + ThousandsOfSingletonMergesRetainChunksAndDestroyIteratively) { + using Sequence = PermutableSequence; + ASSERT_EQ(Tracked::live, 0); + { + Sequence sequence; + std::vector expected; + for (int i = 0; i < 10000; ++i) { + auto input = std::views::iota(i, i + 1) | + std::views::transform([](int n) { return Tracked(n); }); + auto donor = Sequence::from_range(input); + expected.push_back(&donor[0]); + Tracked::moves = 0; + Sequence::test_fail_payload_after(0); + ResetFailures reset; + sequence.merge(donor); + EXPECT_EQ(Tracked::moves, 0); + EXPECT_TRUE(donor.empty()); + } + check_addresses(sequence, expected); + const auto memory = sequence.memory_usage(); + EXPECT_EQ(memory.chunks, 10000); + EXPECT_EQ(memory.vector_live_bytes, 10000 * sizeof(Tracked)); + EXPECT_EQ(memory.vector_slack_bytes, 0); + } + EXPECT_EQ(Tracked::live, 0); +} + +TEST(PermutableSequence, + EveryFailedIndirectMergeAllocationPreservesBothOwners) { + using Tree = Indirect::test_tree_type; + ResetFailures reset; + ASSERT_EQ(Tracked::live, 0); + bool succeeded = false; + for (std::ptrdiff_t failure = 0; failure < 128 && !succeeded; ++failure) { + Tree::test_fail_after(-1); + auto sequence = tracked_sequence(0, 111); + auto donor = tracked_sequence(111, 194); + sequence.rotate_left(2, 109, 17); + donor.rotate_left(0, donor.size(), 23); + const auto original = addresses(sequence); + const auto addition = addresses(donor); + const auto live_allocations = Tree::test_counters.live_allocations; + const auto before = sequence.memory_usage(); + const auto donor_before = donor.memory_usage(); + Tracked::moves = 0; + Tree::test_fail_after(failure); + Indirect::test_fail_payload_after(0); + try { + sequence.merge(donor); + succeeded = true; + auto expected = original; + expected.insert(expected.end(), addition.begin(), addition.end()); + check_addresses(sequence, expected); + EXPECT_TRUE(donor.empty()); + EXPECT_EQ(donor.memory_usage_bytes(), sizeof(donor)); + } catch (const std::bad_alloc&) { + check_addresses(sequence, original); + check_addresses(donor, addition); + EXPECT_EQ(sequence.memory_usage_bytes(), before.total_bytes); + EXPECT_EQ(donor.memory_usage_bytes(), donor_before.total_bytes); + EXPECT_EQ(Tree::test_counters.live_allocations, live_allocations); + } + EXPECT_EQ(Tracked::moves, 0); + EXPECT_EQ(Tracked::live, 194); + Tree::test_fail_after(-1); + Indirect::test_fail_payload_after(-1); + } + EXPECT_TRUE(succeeded); + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TEST(PermutableSequence, + EveryFailedIndirectRotationAllocationPreservesAddresses) { + using Tree = Indirect::test_tree_type; + ResetFailures reset; + bool succeeded = false; + for (std::ptrdiff_t failure = 0; failure < 256 && !succeeded; ++failure) { + Tree::test_fail_after(-1); + auto sequence = tracked_sequence(0, 113); + const auto original = addresses(sequence); + const auto live_allocations = Tree::test_counters.live_allocations; + Tracked::moves = 0; + Tree::test_fail_after(failure); + try { + sequence.rotate_left(1, 111, 41); + succeeded = true; + auto expected = original; + rotate(expected, 1, 111, 41); + check_addresses(sequence, expected); + } catch (const std::bad_alloc&) { + check_addresses(sequence, original); + EXPECT_EQ(Tree::test_counters.live_allocations, live_allocations); + } + EXPECT_EQ(Tracked::moves, 0); + EXPECT_EQ(Tracked::live, 113); + } + EXPECT_TRUE(succeeded); + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TEST(PermutableSequence, EveryFailedPackedMergeAllocationPreservesValues) { + using Sequence = + PermutableSequence; + using Tree = Sequence::test_tree_type; + ResetFailures reset; + bool succeeded = false; + for (std::ptrdiff_t failure = 0; failure < 128 && !succeeded; ++failure) { + Tree::test_fail_after(-1); + auto input = values(111); + auto addition = values(83, 73); + auto sequence = Sequence::from_range(input); + auto donor = Sequence::from_range(addition); + Tree::test_fail_after(failure); + const auto live = Tree::test_counters.live_allocations; + try { + sequence.merge(donor); + succeeded = true; + input.insert(input.end(), addition.begin(), addition.end()); + check(sequence, input); + check(donor, {}); + } catch (const std::bad_alloc&) { + check(sequence, input); + check(donor, addition); + EXPECT_EQ(Tree::test_counters.live_allocations, live); + } + } + EXPECT_TRUE(succeeded); + EXPECT_EQ(Tree::test_counters.live_allocations, 0); +} + +TEST(PermutableSequence, FittingMergeSucceedsWithEveryNextAllocationDisabled) { + using Tree = Indirect::test_tree_type; + ResetFailures reset; + auto sequence = tracked_sequence(0, 1); + auto donor = tracked_sequence(1, 2); + const auto* first = &sequence[0]; + const auto* second = &donor[0]; + Tracked::moves = 0; + Tree::test_fail_after(0); + Indirect::test_fail_payload_after(0); + EXPECT_NO_THROW(sequence.merge(donor)); + EXPECT_EQ(&sequence[0], first); + EXPECT_EQ(&sequence[1], second); + EXPECT_EQ(Tracked::moves, 0); + EXPECT_TRUE(donor.empty()); +} + +TEST(PermutableSequence, ConstructorChunkAndVectorFailuresReclaimAllPayload) { + ResetFailures reset; + ASSERT_EQ(Tracked::live, 0); + // 19 values at four values per chunk: exactly five new/reserve pairs. + for (std::ptrdiff_t failure = 0; failure < 10; ++failure) { + Indirect::test_fail_payload_after(failure); + EXPECT_THROW(tracked_sequence(0, 19), std::bad_alloc); + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Indirect::test_tree_type::test_counters.live_allocations, 0); + } + Indirect::test_fail_payload_after(10); + EXPECT_NO_THROW(tracked_sequence(0, 19)); + EXPECT_EQ(Tracked::live, 0); +} + +TEST(PermutableSequence, EveryConstructorOrderAllocationFailureReclaimsChunks) { + using Tree = Indirect::test_tree_type; + ResetFailures reset; + bool succeeded = false; + for (std::ptrdiff_t failure = 0; failure < 256 && !succeeded; ++failure) { + Tree::test_fail_after(failure); + try { + auto sequence = tracked_sequence(0, 113); + EXPECT_EQ(sequence.size(), 113); + succeeded = true; + } catch (const std::bad_alloc&) { + } + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Tree::test_counters.live_allocations, 0); + } + EXPECT_TRUE(succeeded); +} + +struct CopyThrows { + inline static int live = 0; + inline static int copies_before_throw = -1; + int value; + explicit CopyThrows(int n) : value(n) { ++live; } + CopyThrows(const CopyThrows& other) : value(other.value) { + if (copies_before_throw == 0) { + throw std::runtime_error("input copy"); + } + if (copies_before_throw > 0) { + --copies_before_throw; + } + ++live; + } + CopyThrows(CopyThrows&& other) noexcept : value(other.value) { ++live; } + ~CopyThrows() noexcept { --live; } +}; + +TEST(PermutableSequence, + ThrowingInputAndElementCopiesCleanUpPartialConstruction) { + for (int failure = 0; failure < 23; ++failure) { + auto input = std::views::iota(0, 23) | std::views::transform([&](int n) { + if (n == failure) { + throw std::runtime_error("input iterator"); + } + return Tracked(n); + }); + EXPECT_THROW(Indirect::from_range(input), std::runtime_error); + EXPECT_EQ(Tracked::live, 0); + EXPECT_EQ(Indirect::test_tree_type::test_counters.live_allocations, 0); + } + using Sequence = PermutableSequence; + { + std::vector input; + input.reserve(23); + for (int i = 0; i < 23; ++i) { + input.emplace_back(i); + } + for (int failure = 0; failure < 23; ++failure) { + CopyThrows::copies_before_throw = failure; + EXPECT_THROW(Sequence::from_range(std::as_const(input)), + std::runtime_error); + EXPECT_EQ(CopyThrows::live, 23); + EXPECT_EQ(Sequence::test_tree_type::test_counters.live_allocations, 0); + } + CopyThrows::copies_before_throw = -1; + auto sequence = Sequence::from_range(std::as_const(input)); + EXPECT_EQ(sequence.size(), 23); + } + EXPECT_EQ(CopyThrows::live, 0); +} + +} // namespace diff --git a/src/tests/permutation_tests.cpp b/src/tests/permutation_tests.cpp new file mode 100644 index 0000000..1b3f951 --- /dev/null +++ b/src/tests/permutation_tests.cpp @@ -0,0 +1,522 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using namespace pixie; +using namespace pixie::detail::sequence; + +template +[[gnu::noinline]] void rotate_values(std::vector& values, + std::size_t left, + std::size_t right, + std::size_t distance) { + if (left != right) { + std::rotate(values.begin() + left, + values.begin() + left + distance % (right - left), + values.begin() + right); + } +} + +template +auto contents(const Container& container) { + std::vector result; + for (std::size_t i = 0; i < container.size(); ++i) { + result.push_back(container[i]); + } + return result; +} + +template +void check_permutation(const P& permutation, + const std::vector& expected) { + ASSERT_TRUE(permutation.test_tree().test_validate()); + ASSERT_EQ(permutation.size(), expected.size()); + ASSERT_EQ(permutation.empty(), expected.empty()); + EXPECT_EQ(contents(permutation), expected); + std::vector seen(permutation.size()); + for (std::size_t i = 0; i < permutation.size(); ++i) { + const auto value = permutation[i]; + ASSERT_LT(value, permutation.size()); + ASSERT_FALSE(seen[value]); + seen[value] = true; + } + const auto memory = permutation.memory_usage(); + EXPECT_EQ(memory.total_bytes, + memory.facade_bytes + memory.payload_capacity_bytes + + memory.block_metadata_bytes + memory.ordering_tree_bytes + + memory.tag_padding_bytes); + EXPECT_EQ(memory.total_bytes, permutation.memory_usage_bytes()); + EXPECT_EQ(memory.total_bytes, permutation.test_tree().memory_usage_bytes()); +} + +template +class PermutationSpec : public ::testing::Test {}; +using Permutations = ::testing::Types< + Permutation, + Permutation, + Permutation, + Permutation, + Permutation, + Permutation>; +TYPED_TEST_SUITE(PermutationSpec, Permutations); + +TYPED_TEST(PermutationSpec, PublicContractAliasesNoexceptAndLayout) { + using P = TypeParam; + using T = typename P::value_type; + using Base = PermutationBase; + using Tree = + std::remove_cvref_t().test_tree())>; + static_assert(std::is_base_of_v); + static_assert(std::is_empty_v); + static_assert(std::same_as); + static_assert(std::same_as); + static_assert(std::same_as); + static_assert(std::same_as()[0]), T>); + static_assert(noexcept(std::declval().size())); + static_assert(noexcept(std::declval().empty())); + static_assert(noexcept(std::declval().memory_usage_bytes())); + static_assert(!noexcept(std::declval()[0])); + static_assert(sizeof(P) == sizeof(Tree)); + static_assert(alignof(P) == alignof(Tree)); + auto owner = Base::identity(3); + Base& contract = owner; + contract.rotate_left(0, 3, 1); + auto donor = Base::identity(2); + contract.merge(donor); + EXPECT_EQ(contents(owner), (std::vector{1, 2, 0, 3, 4})); + EXPECT_THROW(contract[contract.size()], std::out_of_range); + EXPECT_EQ(contract.memory_usage_bytes(), owner.memory_usage().total_bytes); +} + +TYPED_TEST(PermutationSpec, IdentityOwnershipExactMergeAndRanges) { + using P = TypeParam; + using T = typename P::value_type; + using Tree = + std::remove_cvref_t().test_tree())>; + static_assert(!std::is_copy_constructible_v

); + static_assert(std::is_nothrow_move_constructible_v

); + static_assert(std::is_nothrow_move_assignable_v

); + P empty; + check_permutation(empty, {}); + auto zero = P::identity(0); + check_permutation(zero, {}); + empty.rotate_left(0, 0, -1); + EXPECT_THROW(empty[0], std::out_of_range); + EXPECT_THROW(empty.rotate_left(0, 1, 0), std::out_of_range); + EXPECT_THROW(empty.rotate_left(1, 0, 0), std::out_of_range); + auto a = P::identity(3); + auto b = P::identity(2); + a.rotate_left(1, 3, 1); + b.rotate_left(0, 2, 1); + Tree::test_reset_counters(); + Tree::test_fail_after(0); + a.merge(b); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, 0); + check_permutation(a, {0, 2, 1, 4, 3}); + check_permutation(b, {}); + a.merge(a); + a.merge(empty); + EXPECT_THROW(a.rotate_left(0, 6, 0), std::out_of_range); + EXPECT_THROW(a.rotate_left(3, 2, 0), std::out_of_range); + EXPECT_THROW(a[5], std::out_of_range); + auto& alias = a; + a = std::move(alias); + P moved(std::move(a)); + check_permutation(a, {}); + b = std::move(moved); + check_permutation(moved, {}); + empty.merge(b); + check_permutation(b, {}); + check_permutation(empty, {0, 2, 1, 4, 3}); + const auto n = Tree::block_capacity * 23 + 1; + auto identity = P::identity(n); + std::vector expected(n); + std::iota(expected.begin(), expected.end(), T{0}); + check_permutation(identity, expected); +} + +TYPED_TEST(PermutationSpec, EveryShortRotation) { + using P = TypeParam; + using T = typename P::value_type; + for (std::size_t n = 0; n <= 8; ++n) { + for (std::size_t left = 0; left <= n; ++left) { + for (std::size_t right = left; right <= n; ++right) { + for (std::size_t d = 0; d <= right - left + 1; ++d) { + auto permutation = P::identity(n); + std::vector expected(n); + std::iota(expected.begin(), expected.end(), T{0}); + permutation.rotate_left(left, right, d); + rotate_values(expected, left, right, d); + check_permutation(permutation, expected); + } + } + } + } +} + +TYPED_TEST(PermutationSpec, TwoLeafSpillPreflightsBeforeRebasing) { + using P = TypeParam; + using Tree = + std::remove_cvref_t().test_tree())>; + for (std::size_t fail = 0; fail <= 1; ++fail) { + auto a = P::identity(Tree::block_capacity); + auto b = P::identity(1); + const auto before = contents(a); + const auto biases_a = a.test_tree().test_bias_snapshot(); + const auto biases_b = b.test_tree().test_bias_snapshot(); + Tree::test_reset_counters(); + Tree::test_fail_after(fail); + if (fail == 0) { + EXPECT_THROW(a.merge(b), std::bad_alloc); + } else { + EXPECT_NO_THROW(a.merge(b)); + } + Tree::test_fail_after(-1); + if (fail == 0) { + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(a.test_tree().test_bias_snapshot(), biases_a); + EXPECT_EQ(b.test_tree().test_bias_snapshot(), biases_b); + check_permutation(a, before); + check_permutation(b, {0}); + } else { + auto expected = before; + expected.push_back(static_cast(before.size())); + check_permutation(a, expected); + check_permutation(b, {}); + EXPECT_EQ(Tree::test_counters.allocations, 1); + } + } +} + +TYPED_TEST(PermutationSpec, RebasedChildRotationAndFailedPreflightKeepRawTags) { + using P = TypeParam; + using Tree = + std::remove_cvref_t().test_tree())>; + const auto make = [] { + auto a = P::identity(83 * Tree::block_capacity); + auto b = P::identity(131 * Tree::block_capacity); + a.merge(b); + auto prefix = P::identity(47 * Tree::block_capacity); + prefix.merge(a); + return prefix; + }; + auto permutation = make(); + auto expected = contents(permutation); + auto biases = permutation.test_tree().test_bias_snapshot(); + ASSERT_TRUE(std::ranges::any_of( + biases, [](const auto& entry) { return entry.second != 0; })); + const auto by_address = [](const auto& a, const auto& b) { + return std::less{}(a.first, b.first); + }; + std::ranges::sort(biases, by_address); + const auto shapes = permutation.test_tree().test_child_boundaries(); + for (auto index : {std::size_t{0}, shapes.size() / 2, shapes.size() - 1}) { + const auto& boundaries = shapes[index]; + const auto l = boundaries.front(), r = boundaries.back(); + const auto d = boundaries[1] - l; + Tree::test_reset_counters(); + Tree::test_fail_after(0); + EXPECT_NO_THROW(permutation.rotate_left(l, r, d)); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.allocations, 0); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + auto after = permutation.test_tree().test_bias_snapshot(); + std::ranges::sort(after, by_address); + EXPECT_EQ(after, biases); + rotate_values(expected, l, r, d); + check_permutation(permutation, expected); + Tree::test_fail_after(0); + EXPECT_NO_THROW(permutation.rotate_left(l, r, r - l - d)); + Tree::test_fail_after(-1); + rotate_values(expected, l, r, r - l - d); + } + std::size_t allocations; + { + auto probe = make(); + Tree::test_reset_counters(); + probe.rotate_left(1, probe.size() - 1, probe.size() / 3); + allocations = Tree::test_counters.allocations; + } + const auto raw = permutation.test_tree().test_bias_snapshot(); + const auto leaves = permutation.test_tree().test_leaf_identities(); + const auto nodes = permutation.test_tree().test_internal_node_identities(); + for (std::size_t fail = 0; fail < allocations; ++fail) { + Tree::test_reset_counters(); + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_fail_after(fail); + EXPECT_THROW(permutation.rotate_left(1, permutation.size() - 1, + permutation.size() / 3), + std::bad_alloc); + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.live_bytes, bytes); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(permutation.test_tree().test_bias_snapshot(), raw); + EXPECT_EQ(permutation.test_tree().test_leaf_identities(), leaves); + EXPECT_EQ(permutation.test_tree().test_internal_node_identities(), nodes); + check_permutation(permutation, expected); + } +} + +TYPED_TEST(PermutationSpec, NestedRebasesAssociationsAndDeepRotations) { + using P = TypeParam; + using T = typename P::value_type; + using Tree = + std::remove_cvref_t().test_tree())>; + constexpr auto capacity = Tree::block_capacity; + std::mt19937_64 random(91723); + auto permutation = P::identity(capacity * 9 + 1); + auto expected = contents(permutation); + for (std::size_t iteration = 0; iteration < 90; ++iteration) { + auto donor = P::identity(1 + random() % (capacity * 3)); + auto extra = P::identity(1 + random() % (capacity * 2)); + donor.rotate_left(0, donor.size(), random()); + donor.merge(extra); + donor.rotate_left(0, donor.size(), random()); + const auto incoming = contents(donor); + if (iteration % 3 == 0) { + auto next = incoming; + for (auto value : expected) { + next.push_back(static_cast(value + incoming.size())); + } + donor.merge(permutation); + permutation = std::move(donor); + expected = std::move(next); + } else { + const auto offset = expected.size(); + for (auto value : incoming) { + expected.push_back(static_cast(value + offset)); + } + permutation.merge(donor); + check_permutation(donor, {}); + } + check_permutation(permutation, expected); + for (std::size_t j = 0; j < 3; ++j) { + const auto left = j == 0 ? 0 : random() % permutation.size(); + const auto right = + j == 0 ? permutation.size() + : left + random() % (permutation.size() - left + 1); + const auto distance = random(); + const auto h = permutation.test_tree().height(); + Tree::test_reset_counters(); + permutation.rotate_left(left, right, distance); + const auto counts = Tree::test_counters; + EXPECT_LE(counts.allocations, 21 * h + 27); + EXPECT_LE(counts.node_visits, 160 * (h + 4)); + EXPECT_LE(counts.child_transfers, 160 * 16 * (h + 4)); + // Three splits + three seams: <=15 redistributions. Each can normalize + // at most two leaves, so <=45 payload calls, independent of n and h. + EXPECT_LE(counts.payload_mutations, 45); + rotate_values(expected, left, right, distance); + check_permutation(permutation, expected); + } + } +} + +TYPED_TEST(PermutationSpec, + AllocationFailureEveryIdentityMergeAndRotationPoint) { + using P = TypeParam; + using Tree = + std::remove_cvref_t().test_tree())>; + constexpr auto capacity = Tree::block_capacity; + Tree::test_reset_counters(); + { auto probe = P::identity(capacity * 19 + 1); } + const auto construction_allocations = Tree::test_counters.allocations; + for (std::size_t fail = 0; fail <= construction_allocations; ++fail) { + Tree::test_fail_after(fail); + if (fail < construction_allocations) { + EXPECT_THROW(P::identity(capacity * 19 + 1), std::bad_alloc); + } else { + EXPECT_NO_THROW(P::identity(capacity * 19 + 1)); + } + Tree::test_fail_after(-1); + EXPECT_EQ(Tree::test_counters.live_allocations, 0); + EXPECT_EQ(Tree::test_counters.live_bytes, 0); + } + auto make = [](std::size_t n) { + auto result = P::identity(n); + // Rebase already rebased subtrees, then rotate without normalizing every + // off-path tag. Both transaction participants must retain nonzero biases. + for (int round = 0; round < 3; ++round) { + auto prefix = P::identity(capacity * 17); + prefix.merge(result); + result = std::move(prefix); + } + result.rotate_left(0, result.size(), 1); + return result; + }; + for (int operation = 0; operation < 5; ++operation) { + auto apply = [operation](P& a, P& b) { + if (operation < 3) { + a.merge(b); + } else if (operation == 3) { + a.rotate_left(0, a.size(), a.size() / 2 + 1); + } else { + a.rotate_left(1, a.size() - 1, a.size() / 3); + } + }; + const auto left_size = operation == 0 ? capacity : capacity * 29 + 1; + const auto right_size = operation == 1 ? capacity * 31 : capacity + 1; + std::size_t allocations; + { + auto a = make(left_size); + auto b = make(right_size); + Tree::test_reset_counters(); + apply(a, b); + allocations = Tree::test_counters.allocations; + } + for (std::size_t fail = 0; fail <= allocations; ++fail) { + auto a = make(left_size); + auto b = make(right_size); + const auto before_a = contents(a); + const auto before_b = contents(b); + const auto leaves_a = a.test_tree().test_leaf_identities(); + const auto nodes_a = a.test_tree().test_internal_node_identities(); + const auto leaves_b = b.test_tree().test_leaf_identities(); + const auto nodes_b = b.test_tree().test_internal_node_identities(); + const auto biases_a = a.test_tree().test_bias_snapshot(); + const auto biases_b = b.test_tree().test_bias_snapshot(); + ASSERT_TRUE(std::ranges::any_of( + biases_a, [](const auto& entry) { return entry.second != 0; })); + ASSERT_TRUE(std::ranges::any_of( + biases_b, [](const auto& entry) { return entry.second != 0; })); + const auto live = Tree::test_counters.live_allocations; + const auto bytes = Tree::test_counters.live_bytes; + Tree::test_reset_counters(); + Tree::test_fail_after(fail); + if (fail < allocations) { + EXPECT_THROW(apply(a, b), std::bad_alloc); + } else { + EXPECT_NO_THROW(apply(a, b)); + } + Tree::test_fail_after(-1); + if (fail < allocations) { + EXPECT_EQ(Tree::test_counters.live_allocations, live); + EXPECT_EQ(Tree::test_counters.live_bytes, bytes); + EXPECT_EQ(Tree::test_counters.payload_mutations, 0); + EXPECT_EQ(a.test_tree().test_leaf_identities(), leaves_a); + EXPECT_EQ(a.test_tree().test_internal_node_identities(), nodes_a); + EXPECT_EQ(b.test_tree().test_leaf_identities(), leaves_b); + EXPECT_EQ(b.test_tree().test_internal_node_identities(), nodes_b); + EXPECT_EQ(a.test_tree().test_bias_snapshot(), biases_a); + EXPECT_EQ(b.test_tree().test_bias_snapshot(), biases_b); + check_permutation(a, before_a); + check_permutation(b, before_b); + } else { + auto expected = before_a; + if (operation < 3) { + for (auto value : before_b) { + expected.push_back( + static_cast(value + before_a.size())); + } + check_permutation(b, {}); + } else { + const auto left = operation == 3 ? 0 : 1; + const auto right = + operation == 3 ? expected.size() : expected.size() - 1; + const auto distance = + operation == 3 ? expected.size() / 2 + 1 : expected.size() / 3; + rotate_values(expected, left, right, distance); + check_permutation(b, before_b); + } + check_permutation(a, expected); + } + } + } +} + +TEST(PermutationBoundary, FullUint16DomainAndOverflowBeforeAllocation) { + using P = Permutation; + using Tree = + std::remove_cvref_t().test_tree())>; + auto a = P::identity(65535); + auto b = P::identity(1); + a.merge(b); + EXPECT_TRUE(b.empty()); + ASSERT_EQ(a.size(), 65536); + EXPECT_EQ(a[65535], 65535); + a.rotate_left(0, a.size(), 32769); + std::vector expected(65536); + std::iota(expected.begin(), expected.end(), std::uint16_t{0}); + rotate_values(expected, 0, expected.size(), 32769); + check_permutation(a, expected); + b = P::identity(1); + const auto biases_a = a.test_tree().test_bias_snapshot(); + const auto biases_b = b.test_tree().test_bias_snapshot(); + Tree::test_fail_after(0); + EXPECT_THROW(a.merge(b), std::length_error); + EXPECT_THROW(P::identity(65537), std::length_error); + Tree::test_fail_after(-1); + EXPECT_EQ(a.test_tree().test_bias_snapshot(), biases_a); + EXPECT_EQ(b.test_tree().test_bias_snapshot(), biases_b); + check_permutation(a, expected); + check_permutation(b, {0}); + auto entire = P::identity(65536); + EXPECT_EQ(entire[65535], 65535); +} + +TEST(PermutationLayout, OptionalBiasHasNoUntaggedLayoutTax) { + using Block = PackedValueBlock; + static_assert(sizeof(Block) == 256); + static_assert(SequenceTree::node_storage_bytes == 64); + static_assert(SequenceTree::node_storage_bytes == 128); + static_assert(SequenceTree::node_storage_bytes == 256); + static_assert(SequenceTree::block_storage_bytes == sizeof(Block)); + static_assert(Block::capacity * 64 == 1920); + static_assert(SequenceTree::node_storage_bytes == 128); + static_assert(SequenceTree::node_storage_bytes == 320); + static_assert( + SequenceTree, 8, LengthLayout::cumulative, + true>::node_storage_bytes == 192); + static_assert( + SequenceTree, 8, LengthLayout::cumulative, + true>::block_storage_bytes == 320); + // F6 uses existing alignment slack rather than another aligned node line. + // Accounting must also handle a zero incremental node-byte category. + auto permutation = Permutation::identity(100); + check_permutation(permutation, contents(permutation)); +} + +template +concept RawBlockTraversal = + requires(const Tree& tree) { tree.for_each_block([](const auto&) {}); }; +static_assert(RawBlockTraversal>>); +static_assert(!RawBlockTraversal, + 8, + LengthLayout::cumulative, + true>>); + +TEST(PermutationStress, ThousandsOfSingletonMerges) { + using P = Permutation; + P permutation; + for (std::size_t i = 0; i < 3000; ++i) { + auto singleton = P::identity(1); + permutation.merge(singleton); + ASSERT_TRUE(singleton.empty()); + ASSERT_EQ(permutation[i], i); + } + permutation.rotate_left(1, permutation.size() - 1, 701); + std::vector expected(permutation.size()); + std::iota(expected.begin(), expected.end(), std::uint16_t{0}); + rotate_values(expected, 1, expected.size() - 1, 701); + check_permutation(permutation, expected); +} + +} // namespace diff --git a/src/tests/sequence_tree_kernels_tests.cpp b/src/tests/sequence_tree_kernels_tests.cpp new file mode 100644 index 0000000..be8b7f8 --- /dev/null +++ b/src/tests/sequence_tree_kernels_tests.cpp @@ -0,0 +1,137 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using pixie::detail::sequence::node_select; +using pixie::detail::sequence::node_select_scalar; + +constexpr auto kMax = std::numeric_limits::max(); +constexpr auto kTop = std::size_t{1} + << (std::numeric_limits::digits - 1); + +void CheckSelection(std::span ends, std::size_t index) { + const auto expected = + ends.empty() ? 0 + : static_cast( + std::upper_bound(ends.begin(), ends.end(), index) - + ends.begin()); + EXPECT_EQ(node_select_scalar(ends, index), expected) + << "count=" << ends.size() << " index=" << index; + EXPECT_EQ(node_select(ends, index), expected) + << "count=" << ends.size() << " index=" << index; +} + +TEST(SequenceTreeKernels, EmptyNullBacking) { + const std::span ends( + static_cast(nullptr), std::size_t{0}); + for (const auto index : + {std::size_t{0}, std::size_t{1}, kTop - 1, kTop, kMax}) { + CheckSelection(ends, index); + } +} + +TEST(SequenceTreeKernels, ExactExtentCountsAndAlignments) { + for (std::size_t count = 1; count <= 15; ++count) { + for (std::size_t offset = 0; offset < 8; ++offset) { + // Every tested span ends at the allocation boundary: ASan can detect + // unmasked vector/tail overreads even when the starting address is + // skewed. + auto storage = std::make_unique(offset + count); + std::span ends(storage.get() + offset, count); + for (const auto base : {std::size_t{0}, kTop - 32, kMax - 64}) { + for (std::size_t i = 0; i < count; ++i) { + ends[i] = base + 4 * i; + } + for (std::size_t delta = 0; delta <= 64; ++delta) { + CheckSelection(ends, base + delta); + } + CheckSelection(ends, 0); + CheckSelection(ends, kMax); + } + for (const auto value : {std::size_t{0}, kTop, kMax}) { + std::fill(ends.begin(), ends.end(), value); + CheckSelection(ends, value); + if (value != 0) { + CheckSelection(ends, value - 1); + } + if (value != kMax) { + CheckSelection(ends, value + 1); + } + } + } + } +} + +TEST(SequenceTreeKernels, ExhaustiveShortMonotoneEnds) { + for (unsigned mask = 0; mask < 256; ++mask) { + const auto count = static_cast(std::popcount(mask)); + auto storage = count ? std::make_unique(count) : nullptr; + std::span ends(storage.get(), count); + for (const auto base : {std::size_t{0}, kTop - 4, kMax - 7}) { + std::size_t i = 0; + for (unsigned bit = 0; bit < 8; ++bit) { + if ((mask >> bit) & 1u) { + ends[i++] = base + bit; + } + } + for (std::size_t delta = 0; delta < 8; ++delta) { + CheckSelection(ends, base + delta); + } + CheckSelection(ends, 0); + CheckSelection(ends, kMax); + } + } +} + +TEST(SequenceTreeKernels, RandomFullWidthAndRepeatedEnds) { + std::mt19937_64 random(0x5e1ec7); + for (std::size_t count = 1; count <= 15; ++count) { + for (std::size_t trial = 0; trial < 128; ++trial) { + const auto offset = trial % 8; + auto storage = std::make_unique(offset + count); + std::span ends(storage.get() + offset, count); + for (auto& end : ends) { + switch (trial % 4) { + case 0: + end = static_cast(random()); + break; + case 1: + end = kTop - 16 + random() % 33; + break; + case 2: + end = kMax - random() % 64; + break; + default: + end = random() % 8; + break; + } + } + std::sort(ends.begin(), ends.end()); + for (const auto end : ends) { + CheckSelection(ends, end); + if (end != 0) { + CheckSelection(ends, end - 1); + } + if (end != kMax) { + CheckSelection(ends, end + 1); + } + } + CheckSelection(ends, 0); + CheckSelection(ends, kMax); + for (unsigned i = 0; i < 16; ++i) { + CheckSelection(ends, static_cast(random())); + } + } + } +} + +} // namespace