From 97ca437f5fca18346ce36a8d8122f1bdc247745c Mon Sep 17 00:00:00 2001 From: Kai Zhao Date: Tue, 6 Oct 2026 16:40:28 -0400 Subject: [PATCH 1/3] Squashed 'SZ3/SZ3/' content from commit 6141ed78 git-subtree-dir: SZ3/SZ3 git-subtree-split: 6141ed7879ad4812fa1e7d1c77e3459bd4baffc6 --- .clang-format | 164 + .github/workflows/build-pysz-test.yml | 108 + .github/workflows/build-pysz.yml | 107 + .github/workflows/cmake.yml | 548 ++ .github/workflows/codeql-analysis.yml | 71 + .github/workflows/gromacs.yml | 394 + .github/workflows/hdf5.yml | 189 + .github/workflows/integration_test.yml | 111 + .github/workflows/libpressio.yml | 79 + .github/workflows/packaging.yml | 369 + .github/workflows/stale.yml | 17 + .gitignore | 23 + CHANGELOG.md | 336 + CMakeLists.txt | 145 + README.md | 108 + SZ3Config.cmake.in | 76 + copyright-and-BSD-license.txt | 29 + default.nix | 36 + docs/advanced-guide.md | 301 + docs/molecular-dynamics.md | 71 + include/SZ3/api/impl/SZAlgoBioMD.hpp | 63 + include/SZ3/api/impl/SZAlgoInterp.hpp | 293 + include/SZ3/api/impl/SZAlgoLorenzoReg.hpp | 88 + include/SZ3/api/impl/SZAlgoNopred.hpp | 36 + include/SZ3/api/impl/SZDispatcher.hpp | 128 + include/SZ3/api/impl/SZImpl.hpp | 93 + include/SZ3/api/impl/SZImplOMP.hpp | 274 + include/SZ3/api/sz.hpp | 204 + include/SZ3/compressor/Compressor.hpp | 40 + .../SZ3/compressor/SZGenericCompressor.hpp | 109 + .../specialized/SZExaaltCompressor.hpp | 218 + .../decomposition/BlockwiseDecomposition.hpp | 106 + include/SZ3/decomposition/Decomposition.hpp | 68 + .../InterpolationDecomposition.hpp | 507 ++ .../NoPredictionDecomposition.hpp | 59 + .../decomposition/SZBioMDDecomposition.hpp | 796 ++ .../decomposition/TimeSeriesDecomposition.hpp | 162 + include/SZ3/def.hpp | 40 + include/SZ3/deprecated/HuffmanEncoderV1.hpp | 244 + include/SZ3/encoder/BypassEncoder.hpp | 48 + include/SZ3/encoder/Encoder.hpp | 78 + include/SZ3/encoder/HuffmanEncoder.hpp | 601 ++ include/SZ3/encoder/SegmentedEncoder.hpp | 132 + include/SZ3/lossless/Lossless.hpp | 45 + include/SZ3/lossless/Lossless_bypass.hpp | 57 + include/SZ3/lossless/Lossless_zstd.hpp | 101 + include/SZ3/predictor/ComposedPredictor.hpp | 135 + include/SZ3/predictor/LorenzoPredictor.hpp | 118 + include/SZ3/predictor/Predictor.hpp | 76 + include/SZ3/predictor/RegressionPredictor.hpp | 187 + include/SZ3/quantizer/LinearQuantizer.hpp | 145 + include/SZ3/quantizer/Quantizer.hpp | 76 + include/SZ3/utils/BlockwiseIterator.hpp | 342 + include/SZ3/utils/ByteUtil.hpp | 340 + include/SZ3/utils/Config.hpp | 516 ++ include/SZ3/utils/Extraction.hpp | 250 + include/SZ3/utils/FileUtil.hpp | 81 + include/SZ3/utils/Interpolators.hpp | 114 + include/SZ3/utils/Iterator.hpp | 322 + include/SZ3/utils/KmeansUtil.hpp | 368 + include/SZ3/utils/MemoryUtil.hpp | 149 + include/SZ3/utils/QuantOptimization.hpp | 158 + include/SZ3/utils/Sample.hpp | 294 + include/SZ3/utils/Statistic.hpp | 157 + include/SZ3/utils/Timer.hpp | 43 + include/SZ3/version.hpp.in | 40 + tools/H5Z-SZ3/CMakeLists.txt | 75 + tools/H5Z-SZ3/README.md | 135 + tools/H5Z-SZ3/include/H5Z_SZ3.hpp | 67 + tools/H5Z-SZ3/src/H5Z_SZ3.cpp | 382 + tools/H5Z-SZ3/test/CMakeLists.txt | 17 + tools/H5Z-SZ3/test/filterSuite.sh | 435 + tools/H5Z-SZ3/test/runFilterSuite.cmake | 12 + tools/H5Z-SZ3/test/sz3_data_3.3.2.h5 | Bin 0 -> 16337 bytes tools/H5Z-SZ3/tools/CMakeLists.txt | 10 + tools/H5Z-SZ3/tools/cdvalueHelper.cpp | 138 + tools/H5Z-SZ3/tools/cdvalueHelper.py | 129 + tools/H5Z-SZ3/tools/convertBinToHDF5.cpp | 202 + tools/H5Z-SZ3/tools/dsz3FromHDF5.cpp | 300 + tools/H5Z-SZ3/tools/sz3ToHDF5.cpp | 232 + tools/mdz/CMakeLists.txt | 20 + tools/mdz/README.md | 37 + tools/mdz/include/mdz.hpp | 508 ++ tools/mdz/mdz.cpp | 89 + tools/mdz/mdz_smoke_test.cpp | 31 + tools/paraview/CMakeLists.txt | 27 + tools/paraview/README.md | 61 + tools/paraview/SZ3Reader/CMakeLists.txt | 13 + .../paraview/SZ3Reader/Reader/CMakeLists.txt | 12 + tools/paraview/SZ3Reader/Reader/SZ3Reader.xml | 48 + tools/paraview/SZ3Reader/Reader/vtk.module | 11 + .../SZ3Reader/Reader/vtkSZ3Reader.cxx | 150 + .../paraview/SZ3Reader/Reader/vtkSZ3Reader.h | 45 + tools/paraview/SZ3Reader/paraview.plugin | 10 + tools/pysz/.gitignore | 32 + tools/pysz/LICENSE | 36 + tools/pysz/MANIFEST.in | 28 + tools/pysz/Makefile | 34 + tools/pysz/README.md | 263 + tools/pysz/deprecated/cpysz.py | 133 + tools/pysz/deprecated/test.py | 25 + tools/pysz/pyproject.toml | 47 + tools/pysz/setup.py | 166 + tools/pysz/src/pysz/__init__.py | 3 + tools/pysz/src/pysz/sz.pxd | 66 + tools/pysz/src/pysz/sz.pyx | 433 + tools/pysz/tests/__init__.py | 1 + tools/pysz/tests/test_pysz.py | 117 + tools/sz3/CMakeLists.txt | 16 + tools/sz3/sz3.config | 48 + tools/sz3/sz3.cpp | 542 ++ tools/sz3/sz3_customized_demo.cpp | 181 + tools/sz3/sz3_smoke_test.cpp | 50 + tools/sz3/testfloat_8_8_128.dat | Bin 0 -> 32768 bytes tools/sz3c/CMakeLists.txt | 30 + tools/sz3c/include/sz3c.h | 64 + tools/sz3c/src/sz3c.cpp | 118 + tools/test/CMakeLists.txt | 89 + tools/test/biomd_finite_math.cpp | 31 + tools/test/bundledZstdIsolation.sh | 157 + tools/test/checkMaxError.sh | 41 + tools/test/check_workflow_env.py | 190 + tools/test/consumeInstalledSZ3.sh | 412 + tools/test/crossBuildDecode.sh | 79 + tools/test/data_format_digest.txt | 10 + tools/test/gromacsConsumerMatrix.sh | 222 + tools/test/gromacsH5mdRoundtrip.sh | 264 + tools/test/integration/datasets.json | 155 + tools/test/integration/deprecated/test_h5.py | 287 + tools/test/integration/deprecated/test_sz3.py | 146 + tools/test/integration/fetch_md_trajectory.py | 211 + .../integration/integration_test_driver.py | 309 + tools/test/integration/test_h5_filter.py | 255 + tools/test/integration/test_sz3_executable.py | 200 + .../update_compression_baseline.py | 68 + tools/test/modules/test_biomd.cpp | 228 + tools/test/modules/test_biomd_codec.cpp | 614 ++ tools/test/modules/test_byte_util.cpp | 69 + tools/test/modules/test_config.cpp | 50 + .../modules/test_decomposition_save_load.cpp | 244 + tools/test/modules/test_encoder.cpp | 48 + tools/test/modules/test_huffman_encoder.cpp | 435 + tools/test/modules/test_integer_data.cpp | 70 + tools/test/modules/test_lossless.cpp | 46 + .../modules/test_previous_data_version.cpp | 101 + tools/test/modules/test_quantizer.cpp | 67 + tools/test/modules/test_reconstruction.cpp | 123 + tools/test/modules/test_segmented_encoder.cpp | 89 + tools/test/openmpThreadCounts.sh | 96 + tools/test/openmp_regions.cpp | 72 + tools/test/openmp_stream.cpp | 152 + tools/test/streams/3.3.2/biomd.sz | Bin 0 -> 870 bytes tools/test/streams/3.3.2/biomdxtc.sz | Bin 0 -> 8949 bytes tools/test/streams/3.3.2/interp.sz | Bin 0 -> 1740 bytes tools/test/streams/3.3.2/interp_omp.sz | Bin 0 -> 4843 bytes tools/test/streams/3.3.2/lorenzo_reg.sz | Bin 0 -> 2010 bytes tools/test/streams/3.3.2/lorenzo_reg_omp.sz | Bin 0 -> 4735 bytes .../test/streams/3.3.2/lorenzo_reg_rel1e-5.sz | Bin 0 -> 3332 bytes tools/test/streams/3.3.2/lossless.sz | Bin 0 -> 735 bytes tools/test/streams/3.3.2/nopred.sz | Bin 0 -> 917 bytes tools/test/sz3LinkTokens.sh | 68 + tools/zstd/CMakeLists.txt | 73 + tools/zstd/COPYING | 339 + tools/zstd/LICENSE | 30 + tools/zstd/lib/common/allocations.h | 55 + tools/zstd/lib/common/bits.h | 200 + tools/zstd/lib/common/bitstream.h | 457 ++ tools/zstd/lib/common/compiler.h | 450 ++ tools/zstd/lib/common/cpu.h | 249 + tools/zstd/lib/common/debug.c | 30 + tools/zstd/lib/common/debug.h | 116 + tools/zstd/lib/common/entropy_common.c | 340 + tools/zstd/lib/common/error_private.c | 63 + tools/zstd/lib/common/error_private.h | 168 + tools/zstd/lib/common/fse.h | 640 ++ tools/zstd/lib/common/fse_decompress.c | 313 + tools/zstd/lib/common/huf.h | 286 + tools/zstd/lib/common/mem.h | 426 + tools/zstd/lib/common/pool.c | 371 + tools/zstd/lib/common/pool.h | 90 + tools/zstd/lib/common/portability_macros.h | 158 + tools/zstd/lib/common/threading.c | 182 + tools/zstd/lib/common/threading.h | 150 + tools/zstd/lib/common/xxhash.c | 18 + tools/zstd/lib/common/xxhash.h | 7020 ++++++++++++++++ tools/zstd/lib/common/zstd_common.c | 48 + tools/zstd/lib/common/zstd_deps.h | 111 + tools/zstd/lib/common/zstd_internal.h | 392 + tools/zstd/lib/common/zstd_trace.h | 163 + tools/zstd/lib/compress/clevels.h | 134 + tools/zstd/lib/compress/fse_compress.c | 625 ++ tools/zstd/lib/compress/hist.c | 181 + tools/zstd/lib/compress/hist.h | 75 + tools/zstd/lib/compress/huf_compress.c | 1464 ++++ tools/zstd/lib/compress/zstd_compress.c | 7153 +++++++++++++++++ .../lib/compress/zstd_compress_internal.h | 1534 ++++ .../lib/compress/zstd_compress_literals.c | 235 + .../lib/compress/zstd_compress_literals.h | 39 + .../lib/compress/zstd_compress_sequences.c | 442 + .../lib/compress/zstd_compress_sequences.h | 54 + .../lib/compress/zstd_compress_superblock.c | 688 ++ .../lib/compress/zstd_compress_superblock.h | 32 + tools/zstd/lib/compress/zstd_cwksp.h | 748 ++ tools/zstd/lib/compress/zstd_double_fast.c | 770 ++ tools/zstd/lib/compress/zstd_double_fast.h | 50 + tools/zstd/lib/compress/zstd_fast.c | 968 +++ tools/zstd/lib/compress/zstd_fast.h | 38 + tools/zstd/lib/compress/zstd_lazy.c | 2199 +++++ tools/zstd/lib/compress/zstd_lazy.h | 202 + tools/zstd/lib/compress/zstd_ldm.c | 730 ++ tools/zstd/lib/compress/zstd_ldm.h | 117 + tools/zstd/lib/compress/zstd_ldm_geartab.h | 106 + tools/zstd/lib/compress/zstd_opt.c | 1576 ++++ tools/zstd/lib/compress/zstd_opt.h | 80 + tools/zstd/lib/compress/zstdmt_compress.c | 1882 +++++ tools/zstd/lib/compress/zstdmt_compress.h | 113 + tools/zstd/lib/decompress/huf_decompress.c | 1944 +++++ .../lib/decompress/huf_decompress_amd64.S | 595 ++ tools/zstd/lib/decompress/zstd_ddict.c | 244 + tools/zstd/lib/decompress/zstd_ddict.h | 44 + tools/zstd/lib/decompress/zstd_decompress.c | 2407 ++++++ .../lib/decompress/zstd_decompress_block.c | 2215 +++++ .../lib/decompress/zstd_decompress_block.h | 73 + .../lib/decompress/zstd_decompress_internal.h | 240 + tools/zstd/lib/zstd.h | 3089 +++++++ tools/zstd/lib/zstd_errors.h | 114 + 226 files changed, 69248 insertions(+) create mode 100644 .clang-format create mode 100644 .github/workflows/build-pysz-test.yml create mode 100644 .github/workflows/build-pysz.yml create mode 100644 .github/workflows/cmake.yml create mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/gromacs.yml create mode 100644 .github/workflows/hdf5.yml create mode 100644 .github/workflows/integration_test.yml create mode 100644 .github/workflows/libpressio.yml create mode 100644 .github/workflows/packaging.yml create mode 100644 .github/workflows/stale.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 SZ3Config.cmake.in create mode 100644 copyright-and-BSD-license.txt create mode 100644 default.nix create mode 100644 docs/advanced-guide.md create mode 100644 docs/molecular-dynamics.md create mode 100644 include/SZ3/api/impl/SZAlgoBioMD.hpp create mode 100644 include/SZ3/api/impl/SZAlgoInterp.hpp create mode 100644 include/SZ3/api/impl/SZAlgoLorenzoReg.hpp create mode 100644 include/SZ3/api/impl/SZAlgoNopred.hpp create mode 100644 include/SZ3/api/impl/SZDispatcher.hpp create mode 100644 include/SZ3/api/impl/SZImpl.hpp create mode 100644 include/SZ3/api/impl/SZImplOMP.hpp create mode 100644 include/SZ3/api/sz.hpp create mode 100644 include/SZ3/compressor/Compressor.hpp create mode 100644 include/SZ3/compressor/SZGenericCompressor.hpp create mode 100644 include/SZ3/compressor/specialized/SZExaaltCompressor.hpp create mode 100644 include/SZ3/decomposition/BlockwiseDecomposition.hpp create mode 100644 include/SZ3/decomposition/Decomposition.hpp create mode 100644 include/SZ3/decomposition/InterpolationDecomposition.hpp create mode 100644 include/SZ3/decomposition/NoPredictionDecomposition.hpp create mode 100644 include/SZ3/decomposition/SZBioMDDecomposition.hpp create mode 100644 include/SZ3/decomposition/TimeSeriesDecomposition.hpp create mode 100644 include/SZ3/def.hpp create mode 100644 include/SZ3/deprecated/HuffmanEncoderV1.hpp create mode 100644 include/SZ3/encoder/BypassEncoder.hpp create mode 100644 include/SZ3/encoder/Encoder.hpp create mode 100644 include/SZ3/encoder/HuffmanEncoder.hpp create mode 100644 include/SZ3/encoder/SegmentedEncoder.hpp create mode 100644 include/SZ3/lossless/Lossless.hpp create mode 100644 include/SZ3/lossless/Lossless_bypass.hpp create mode 100644 include/SZ3/lossless/Lossless_zstd.hpp create mode 100644 include/SZ3/predictor/ComposedPredictor.hpp create mode 100644 include/SZ3/predictor/LorenzoPredictor.hpp create mode 100644 include/SZ3/predictor/Predictor.hpp create mode 100644 include/SZ3/predictor/RegressionPredictor.hpp create mode 100644 include/SZ3/quantizer/LinearQuantizer.hpp create mode 100644 include/SZ3/quantizer/Quantizer.hpp create mode 100644 include/SZ3/utils/BlockwiseIterator.hpp create mode 100644 include/SZ3/utils/ByteUtil.hpp create mode 100644 include/SZ3/utils/Config.hpp create mode 100644 include/SZ3/utils/Extraction.hpp create mode 100644 include/SZ3/utils/FileUtil.hpp create mode 100644 include/SZ3/utils/Interpolators.hpp create mode 100644 include/SZ3/utils/Iterator.hpp create mode 100644 include/SZ3/utils/KmeansUtil.hpp create mode 100644 include/SZ3/utils/MemoryUtil.hpp create mode 100644 include/SZ3/utils/QuantOptimization.hpp create mode 100644 include/SZ3/utils/Sample.hpp create mode 100644 include/SZ3/utils/Statistic.hpp create mode 100644 include/SZ3/utils/Timer.hpp create mode 100644 include/SZ3/version.hpp.in create mode 100644 tools/H5Z-SZ3/CMakeLists.txt create mode 100644 tools/H5Z-SZ3/README.md create mode 100644 tools/H5Z-SZ3/include/H5Z_SZ3.hpp create mode 100644 tools/H5Z-SZ3/src/H5Z_SZ3.cpp create mode 100644 tools/H5Z-SZ3/test/CMakeLists.txt create mode 100755 tools/H5Z-SZ3/test/filterSuite.sh create mode 100644 tools/H5Z-SZ3/test/runFilterSuite.cmake create mode 100644 tools/H5Z-SZ3/test/sz3_data_3.3.2.h5 create mode 100644 tools/H5Z-SZ3/tools/CMakeLists.txt create mode 100644 tools/H5Z-SZ3/tools/cdvalueHelper.cpp create mode 100644 tools/H5Z-SZ3/tools/cdvalueHelper.py create mode 100644 tools/H5Z-SZ3/tools/convertBinToHDF5.cpp create mode 100644 tools/H5Z-SZ3/tools/dsz3FromHDF5.cpp create mode 100644 tools/H5Z-SZ3/tools/sz3ToHDF5.cpp create mode 100644 tools/mdz/CMakeLists.txt create mode 100644 tools/mdz/README.md create mode 100644 tools/mdz/include/mdz.hpp create mode 100644 tools/mdz/mdz.cpp create mode 100644 tools/mdz/mdz_smoke_test.cpp create mode 100644 tools/paraview/CMakeLists.txt create mode 100644 tools/paraview/README.md create mode 100644 tools/paraview/SZ3Reader/CMakeLists.txt create mode 100644 tools/paraview/SZ3Reader/Reader/CMakeLists.txt create mode 100644 tools/paraview/SZ3Reader/Reader/SZ3Reader.xml create mode 100644 tools/paraview/SZ3Reader/Reader/vtk.module create mode 100644 tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.cxx create mode 100644 tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.h create mode 100644 tools/paraview/SZ3Reader/paraview.plugin create mode 100644 tools/pysz/.gitignore create mode 100644 tools/pysz/LICENSE create mode 100644 tools/pysz/MANIFEST.in create mode 100644 tools/pysz/Makefile create mode 100644 tools/pysz/README.md create mode 100644 tools/pysz/deprecated/cpysz.py create mode 100644 tools/pysz/deprecated/test.py create mode 100644 tools/pysz/pyproject.toml create mode 100644 tools/pysz/setup.py create mode 100644 tools/pysz/src/pysz/__init__.py create mode 100644 tools/pysz/src/pysz/sz.pxd create mode 100644 tools/pysz/src/pysz/sz.pyx create mode 100644 tools/pysz/tests/__init__.py create mode 100644 tools/pysz/tests/test_pysz.py create mode 100644 tools/sz3/CMakeLists.txt create mode 100644 tools/sz3/sz3.config create mode 100644 tools/sz3/sz3.cpp create mode 100644 tools/sz3/sz3_customized_demo.cpp create mode 100644 tools/sz3/sz3_smoke_test.cpp create mode 100644 tools/sz3/testfloat_8_8_128.dat create mode 100644 tools/sz3c/CMakeLists.txt create mode 100644 tools/sz3c/include/sz3c.h create mode 100644 tools/sz3c/src/sz3c.cpp create mode 100644 tools/test/CMakeLists.txt create mode 100644 tools/test/biomd_finite_math.cpp create mode 100755 tools/test/bundledZstdIsolation.sh create mode 100755 tools/test/checkMaxError.sh create mode 100755 tools/test/check_workflow_env.py create mode 100755 tools/test/consumeInstalledSZ3.sh create mode 100755 tools/test/crossBuildDecode.sh create mode 100644 tools/test/data_format_digest.txt create mode 100755 tools/test/gromacsConsumerMatrix.sh create mode 100755 tools/test/gromacsH5mdRoundtrip.sh create mode 100644 tools/test/integration/datasets.json create mode 100644 tools/test/integration/deprecated/test_h5.py create mode 100644 tools/test/integration/deprecated/test_sz3.py create mode 100644 tools/test/integration/fetch_md_trajectory.py create mode 100644 tools/test/integration/integration_test_driver.py create mode 100644 tools/test/integration/test_h5_filter.py create mode 100644 tools/test/integration/test_sz3_executable.py create mode 100644 tools/test/integration/update_compression_baseline.py create mode 100644 tools/test/modules/test_biomd.cpp create mode 100644 tools/test/modules/test_biomd_codec.cpp create mode 100644 tools/test/modules/test_byte_util.cpp create mode 100644 tools/test/modules/test_config.cpp create mode 100644 tools/test/modules/test_decomposition_save_load.cpp create mode 100644 tools/test/modules/test_encoder.cpp create mode 100644 tools/test/modules/test_huffman_encoder.cpp create mode 100644 tools/test/modules/test_integer_data.cpp create mode 100644 tools/test/modules/test_lossless.cpp create mode 100644 tools/test/modules/test_previous_data_version.cpp create mode 100644 tools/test/modules/test_quantizer.cpp create mode 100644 tools/test/modules/test_reconstruction.cpp create mode 100644 tools/test/modules/test_segmented_encoder.cpp create mode 100755 tools/test/openmpThreadCounts.sh create mode 100644 tools/test/openmp_regions.cpp create mode 100644 tools/test/openmp_stream.cpp create mode 100644 tools/test/streams/3.3.2/biomd.sz create mode 100644 tools/test/streams/3.3.2/biomdxtc.sz create mode 100644 tools/test/streams/3.3.2/interp.sz create mode 100644 tools/test/streams/3.3.2/interp_omp.sz create mode 100644 tools/test/streams/3.3.2/lorenzo_reg.sz create mode 100644 tools/test/streams/3.3.2/lorenzo_reg_omp.sz create mode 100644 tools/test/streams/3.3.2/lorenzo_reg_rel1e-5.sz create mode 100644 tools/test/streams/3.3.2/lossless.sz create mode 100644 tools/test/streams/3.3.2/nopred.sz create mode 100755 tools/test/sz3LinkTokens.sh create mode 100644 tools/zstd/CMakeLists.txt create mode 100644 tools/zstd/COPYING create mode 100644 tools/zstd/LICENSE create mode 100644 tools/zstd/lib/common/allocations.h create mode 100644 tools/zstd/lib/common/bits.h create mode 100644 tools/zstd/lib/common/bitstream.h create mode 100644 tools/zstd/lib/common/compiler.h create mode 100644 tools/zstd/lib/common/cpu.h create mode 100644 tools/zstd/lib/common/debug.c create mode 100644 tools/zstd/lib/common/debug.h create mode 100644 tools/zstd/lib/common/entropy_common.c create mode 100644 tools/zstd/lib/common/error_private.c create mode 100644 tools/zstd/lib/common/error_private.h create mode 100644 tools/zstd/lib/common/fse.h create mode 100644 tools/zstd/lib/common/fse_decompress.c create mode 100644 tools/zstd/lib/common/huf.h create mode 100644 tools/zstd/lib/common/mem.h create mode 100644 tools/zstd/lib/common/pool.c create mode 100644 tools/zstd/lib/common/pool.h create mode 100644 tools/zstd/lib/common/portability_macros.h create mode 100644 tools/zstd/lib/common/threading.c create mode 100644 tools/zstd/lib/common/threading.h create mode 100644 tools/zstd/lib/common/xxhash.c create mode 100644 tools/zstd/lib/common/xxhash.h create mode 100644 tools/zstd/lib/common/zstd_common.c create mode 100644 tools/zstd/lib/common/zstd_deps.h create mode 100644 tools/zstd/lib/common/zstd_internal.h create mode 100644 tools/zstd/lib/common/zstd_trace.h create mode 100644 tools/zstd/lib/compress/clevels.h create mode 100644 tools/zstd/lib/compress/fse_compress.c create mode 100644 tools/zstd/lib/compress/hist.c create mode 100644 tools/zstd/lib/compress/hist.h create mode 100644 tools/zstd/lib/compress/huf_compress.c create mode 100644 tools/zstd/lib/compress/zstd_compress.c create mode 100644 tools/zstd/lib/compress/zstd_compress_internal.h create mode 100644 tools/zstd/lib/compress/zstd_compress_literals.c create mode 100644 tools/zstd/lib/compress/zstd_compress_literals.h create mode 100644 tools/zstd/lib/compress/zstd_compress_sequences.c create mode 100644 tools/zstd/lib/compress/zstd_compress_sequences.h create mode 100644 tools/zstd/lib/compress/zstd_compress_superblock.c create mode 100644 tools/zstd/lib/compress/zstd_compress_superblock.h create mode 100644 tools/zstd/lib/compress/zstd_cwksp.h create mode 100644 tools/zstd/lib/compress/zstd_double_fast.c create mode 100644 tools/zstd/lib/compress/zstd_double_fast.h create mode 100644 tools/zstd/lib/compress/zstd_fast.c create mode 100644 tools/zstd/lib/compress/zstd_fast.h create mode 100644 tools/zstd/lib/compress/zstd_lazy.c create mode 100644 tools/zstd/lib/compress/zstd_lazy.h create mode 100644 tools/zstd/lib/compress/zstd_ldm.c create mode 100644 tools/zstd/lib/compress/zstd_ldm.h create mode 100644 tools/zstd/lib/compress/zstd_ldm_geartab.h create mode 100644 tools/zstd/lib/compress/zstd_opt.c create mode 100644 tools/zstd/lib/compress/zstd_opt.h create mode 100644 tools/zstd/lib/compress/zstdmt_compress.c create mode 100644 tools/zstd/lib/compress/zstdmt_compress.h create mode 100644 tools/zstd/lib/decompress/huf_decompress.c create mode 100644 tools/zstd/lib/decompress/huf_decompress_amd64.S create mode 100644 tools/zstd/lib/decompress/zstd_ddict.c create mode 100644 tools/zstd/lib/decompress/zstd_ddict.h create mode 100644 tools/zstd/lib/decompress/zstd_decompress.c create mode 100644 tools/zstd/lib/decompress/zstd_decompress_block.c create mode 100644 tools/zstd/lib/decompress/zstd_decompress_block.h create mode 100644 tools/zstd/lib/decompress/zstd_decompress_internal.h create mode 100644 tools/zstd/lib/zstd.h create mode 100644 tools/zstd/lib/zstd_errors.h diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..a5c47981 --- /dev/null +++ b/.clang-format @@ -0,0 +1,164 @@ +--- +BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignConsecutiveMacros: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: WithoutElse +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakAfterJavaFieldAnnotations: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: "^ IWYU pragma:" +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: true +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Regroup +IncludeCategories: + - Regex: ^ + Priority: 2 + SortPriority: 0 + - Regex: ^<.*\.h> + Priority: 1 + SortPriority: 0 + - Regex: ^<.* + Priority: 2 + SortPriority: 0 + - Regex: .* + Priority: 3 + SortPriority: 0 +IncludeIsMainRegex: ([-_](test|unittest))?$ +IncludeIsMainSourceRegex: "" +IndentCaseLabels: true +IndentGotoLabels: true +IndentPPDirectives: None +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +Language: Cpp +MacroBlockBegin: "" +MacroBlockEnd: "" +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - c++ + - C++ + CanonicalDelimiter: "" + BasedOnStyle: google + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: "" + BasedOnStyle: google +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInCStyleCastParentheses: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseCRLF: false +UseTab: Never diff --git a/.github/workflows/build-pysz-test.yml b/.github/workflows/build-pysz-test.yml new file mode 100644 index 00000000..4751a599 --- /dev/null +++ b/.github/workflows/build-pysz-test.yml @@ -0,0 +1,108 @@ +name: Build pysz (TestPyPI) + +on: + push: + tags: + - 'pysz-test-v*' # Trigger on test tags like pysz-test-v1.0.2 + workflow_dispatch: # Allow manual trigger + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + - os: macos-15-intel # Intel + arch: x86_64 + - os: macos-latest # Apple Silicon + arch: arm64 + - os: windows-latest + arch: AMD64 + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install cibuildwheel + run: pip install cibuildwheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + working-directory: tools/pysz + env: + # Build for Python 3.9-3.14 + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* + + # Use manylinux_2_28 (AlmaLinux 8) instead of manylinux2014 (CentOS 7) + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + + # Skip 32-bit builds and musllinux + CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*" + + # Build only for the current architecture + CIBW_ARCHS_LINUX: x86_64 + CIBW_ARCHS_MACOS: ${{ matrix.arch }} + CIBW_ARCHS_WINDOWS: AMD64 + + # Install system dependencies before building + CIBW_BEFORE_ALL_LINUX: yum install -y --nogpgcheck cmake git ninja-build + CIBW_BEFORE_ALL_MACOS: brew install cmake git ninja + + # Test the built wheels + CIBW_TEST_REQUIRES: pytest numpy + CIBW_TEST_COMMAND: "pytest {project}/tests -v" + + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }}-${{ matrix.arch }} + path: tools/pysz/wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build dependencies + run: pip install build + + - name: Build sdist + run: python -m build --sdist + working-directory: tools/pysz + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: tools/pysz/dist/*.tar.gz + + upload_testpypi: + name: Upload to TestPyPI + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/pysz-test-v') + + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + pattern: '*' + merge-multiple: true + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + packages-dir: dist/ diff --git a/.github/workflows/build-pysz.yml b/.github/workflows/build-pysz.yml new file mode 100644 index 00000000..950455c3 --- /dev/null +++ b/.github/workflows/build-pysz.yml @@ -0,0 +1,107 @@ +name: Build pysz + +on: + push: + tags: + - 'pysz-v*' # Trigger on pysz tags like pysz-v1.0.2 + workflow_dispatch: # Allow manual trigger + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + arch: x86_64 + - os: macos-15-intel # Intel + arch: x86_64 + - os: macos-latest # Apple Silicon + arch: arm64 + - os: windows-latest + arch: AMD64 + + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install cibuildwheel + run: pip install cibuildwheel + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + working-directory: tools/pysz + env: + # Build for Python 3.9-3.14 + CIBW_BUILD: cp39-* cp310-* cp311-* cp312-* cp313-* cp314-* + + # Use manylinux_2_28 (AlmaLinux 8) instead of manylinux2014 (CentOS 7) + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + + # Skip 32-bit builds and musllinux + CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_*" + + # Build only for the current architecture + CIBW_ARCHS_LINUX: x86_64 + CIBW_ARCHS_MACOS: ${{ matrix.arch }} + CIBW_ARCHS_WINDOWS: AMD64 + + # Install system dependencies before building + CIBW_BEFORE_ALL_LINUX: yum install -y --nogpgcheck cmake git ninja-build + CIBW_BEFORE_ALL_MACOS: brew install cmake git ninja + + # Test the built wheels + CIBW_TEST_REQUIRES: pytest numpy + CIBW_TEST_COMMAND: "pytest {project}/tests -v" + + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }}-${{ matrix.arch }} + path: tools/pysz/wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build dependencies + run: pip install build + + - name: Build sdist + run: python -m build --sdist + working-directory: tools/pysz + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: tools/pysz/dist/*.tar.gz + + upload_pypi: + name: Upload to PyPI + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/pysz-v') + + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + pattern: '*' + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} + packages-dir: dist/ diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml new file mode 100644 index 00000000..17ec065d --- /dev/null +++ b/.github/workflows/cmake.yml @@ -0,0 +1,548 @@ +name: Build & Test + +on: [ push ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BUILD_TYPE: Release + TEST_URL: https://raw.githubusercontent.com/szcompressor/SZ3/master/tools/sz3/testfloat_8_8_128.dat + DIMS: "128 8 8" + MODE: ABS + TOL: 1 + +# Anything that needs more than one configuration of anything lives in its own workflow: +# packaging.yml for install and consume, hdf5.yml for the filter against more than one HDF5. +jobs: + workflow-env: + name: Every workflow declares the env it reads + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Check for env keys that interpolate to nothing + shell: bash + run: python3 tools/test/check_workflow_env.py + + build-linux: + name: Build & Test (Linux) + runs-on: ubuntu-latest + outputs: + digest: ${{ steps.calc.outputs.digest }} + steps: + - uses: actions/checkout@v5 + + - name: Install HDF5 and Zstd + run: | + sudo apt-get update + # libzstd-dev, not libzstd1: only the -dev package carries the libzstd.pc that sends this + # job down the system-Zstd path. hdf5-tools is what registers SZ3_FilterSuite: the + # test only exists when h5repack, h5dump and h5ls are all found at configure time. + sudo apt-get install -y libhdf5-dev hdf5-tools libzstd-dev + + - name: Create build dir + run: cmake -E make_directory build + + - name: Configure + shell: bash + working-directory: build + run: | + cmake .. \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + shell: bash + working-directory: build + run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Run CTest + shell: bash + working-directory: build + run: ctest --output-on-failure --no-tests=error -C ${{ env.BUILD_TYPE }} + + - name: Download test data + shell: bash + run: | + curl -L -o input.dat ${{ env.TEST_URL }} + + - name: Run sz3 and check error + shell: bash + working-directory: build + run: | + ./tools/sz3/sz3 -f -i ../input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a \ + -o out.dat -z out.sz3 \ + | tee sz3.log + ../tools/test/checkMaxError.sh sz3.log ${{ env.TOL }} + + - name: cmake found OpenMP on this runner + shell: bash + working-directory: build + run: | + # find_package(OpenMP) is not REQUIRED, so a runner without it builds an sz3 that never + # writes an OpenMP stream. Asked here, the two steps below fail for what they are about + # instead of passing on a serial stream. + grep -q '^OpenMP_CXX_FLAGS:STRING=.' CMakeCache.txt \ + || { echo "cmake found no OpenMP on this runner, so the OpenMP path is untested"; exit 1; } + + - name: The OpenMP path stays inside the error bound + shell: bash + working-directory: build + run: | + # The OpenMP path splits the data along the slowest dimension and compresses each piece on + # its own thread, so it has its own stream framing and its own decoder. Nothing else in CI + # enters it. + printf '[GlobalSettings]\nOpenMP = YES\n' > omp.config + ./tools/sz3/sz3 -f -i ../input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a \ + -c omp.config -o omp.dat -z omp.sz3 \ + | tee omp.log + ../tools/test/checkMaxError.sh omp.log ${{ env.TOL }} + + - name: An OpenMP stream decodes the same under any thread count + shell: bash + run: | + tools/test/openmpThreadCounts.sh build/tools/sz3/sz3 input.dat ${{ env.TOL }} \ + -3 ${{ env.DIMS }} + + - name: Compute SHA256 of compressed + id: calc + shell: bash + working-directory: build + run: | + DIGEST=$(sha256sum out.sz3 | awk '{print $1}') + [ -n "$DIGEST" ] || { echo "no digest to compare against macOS"; exit 1; } + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + + - name: Upload Linux-compressed artifact + uses: actions/upload-artifact@v4 + with: + name: temp-linux-out-sz3 + path: build/out.sz3 + + sanitizers: + name: Build & Test (Linux, ASan + UBSan) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Zstd + run: | + sudo apt-get update + sudo apt-get install -y libzstd-dev + + # No HDF5 filter here. The point is SZ3's own writes, and linking HDF5 would report its + # allocations as ours the moment leak detection is on. + - name: Configure + shell: bash + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=OFF \ + -DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=undefined -g" \ + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=undefined -g" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined" + + - name: Build + shell: bash + run: cmake --build build --parallel 4 + + - name: Run CTest + shell: bash + working-directory: build + env: + ASAN_OPTIONS: detect_leaks=1 + run: ctest --output-on-failure --no-tests=error + + # The unit tests check each stage against its own size_est(); this runs the whole CLI at a + # block size where the regression coefficients dominate the buffer. + - name: A small block size does not run off the compressor's buffer + shell: bash + working-directory: build + env: + ASAN_OPTIONS: detect_leaks=1 + run: | + python3 -c " + import struct, random + random.seed(1) + n = 16384 + v = [i * 0.5 + random.uniform(-1, 1) for i in range(n)] + open('ramp.f32', 'wb').write(struct.pack('<%df' % n, *v)) + " + for eb in 1e-1 1e-2 1e-3 1e-4 1e-5; do + for bs in 2 3 4 6; do + printf '[GlobalSettings]\nCmprAlgo = ALGO_LORENZO_REG\nErrorBoundMode = ABS\nAbsErrorBound = %s\n[AlgoSettings]\nBlockSize = %s\n' "$eb" "$bs" > ramp.ini + ./tools/sz3/sz3 -f -i ramp.f32 -z ramp.sz -1 16384 -c ramp.ini + done + done + + fma: + name: Build & Test (Linux, with and without FMA) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Zstd + run: | + sudo apt-get update + sudo apt-get install -y libzstd-dev + + - name: Build + shell: bash + run: | + cmake -S . -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} + cmake --build build --target sz3 --parallel 4 + cmake -S . -B build-fma -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DBUILD_TESTING=ON \ + -DCMAKE_C_FLAGS=-march=x86-64-v3 -DCMAKE_CXX_FLAGS=-march=x86-64-v3 + cmake --build build-fma --target sz3 test_reconstruction --parallel 4 + + # Without FMA instructions in this build, the tests below would compare two identical builds. + - name: The FMA build fuses multiply-adds + shell: bash + run: objdump -d build-fma/tools/sz3/sz3 | grep -q vfmadd + + - name: Both builds decode every stream either writes to the same bytes + shell: bash + run: tools/test/crossBuildDecode.sh build build-fma tools/sz3/testfloat_8_8_128.dat -3 ${{ env.DIMS }} + + - name: The compressor's reconstruction equals the decompressor's output + shell: bash + run: build-fma/tools/test/test_reconstruction + + wasm32: + name: Build & Test (wasm32, 32-bit size_t) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Zstd + run: | + sudo apt-get update + sudo apt-get install -y libzstd-dev + + - name: Install Emscripten + shell: bash + run: | + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$RUNNER_TEMP/emsdk" + "$RUNNER_TEMP/emsdk/emsdk" install 6.0.11 + "$RUNNER_TEMP/emsdk/emsdk" activate 6.0.11 + + # A wasm32 build has a 4-byte size_t. Every size in the stream has to be written as 8 bytes + # for its data and a 64-bit build's to be the same stream. + - name: Build + shell: bash + run: | + cmake -S . -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} + cmake --build build --target sz3 --parallel 4 + source "$RUNNER_TEMP/emsdk/emsdk_env.sh" + # Emscripten's OpenMP is found but does not link; NODERAWFS lets sz3 open files under node. + emcmake cmake -S . -B build-wasm -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DSZ3_USE_BUNDLED_ZSTD=ON -DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=ON \ + -DCMAKE_CXX_FLAGS=-fexceptions \ + -DCMAKE_EXE_LINKER_FLAGS="-fexceptions -sNODERAWFS=1 -sALLOW_MEMORY_GROWTH=1 -sEXIT_RUNTIME=1" + cmake --build build-wasm --target sz3 --parallel 4 + printf '#!/bin/sh\nexec node "$(dirname "$0")/sz3.js" "$@"\n' > build-wasm/tools/sz3/sz3 + chmod +x build-wasm/tools/sz3/sz3 + + - name: Both builds decode every stream either writes to the same bytes + shell: bash + run: tools/test/crossBuildDecode.sh build build-wasm tools/sz3/testfloat_8_8_128.dat -3 ${{ env.DIMS }} + + toolchains: + name: ${{ matrix.cc }} shared=${{ matrix.shared }} + runs-on: ubuntu-latest + # Separate from build-linux, which feeds one digest to compare and one artifact to + # build-windows; a matrix would make both ambiguous. + strategy: + fail-fast: false + matrix: + cc: [gcc-12, gcc-14, clang-16, clang-18] + shared: ["ON", "OFF"] + steps: + - uses: actions/checkout@v5 + + - name: Install the toolchain and HDF5 + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev hdf5-tools libzstd-dev ${{ matrix.cc }} + + - name: Build and test + shell: bash + run: | + # A header-only library sees more of a compiler than most: every template it has is + # instantiated in the consumer's translation unit. BUILD_SHARED_LIBS decides what shape + # hdf5sz3 and SZ3c take, and the static one is what a packager on Windows gets. + CXX=${{ matrix.cc }} + case "$CXX" in clang-*) CXX="clang++-${CXX#clang-}";; gcc-*) CXX="g++-${CXX#gcc-}";; esac + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DCMAKE_C_COMPILER=${{ matrix.cc }} \ + -DCMAKE_CXX_COMPILER="$CXX" \ + -DBUILD_SHARED_LIBS=${{ matrix.shared }} \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=ON + cmake --build build --parallel 4 + (cd build && ctest --output-on-failure --no-tests=error -C ${{ env.BUILD_TYPE }}) + if [ "${{ matrix.shared }}" = "ON" ]; then + test -e build/tools/H5Z-SZ3/libhdf5sz3.so + else + test -e build/tools/H5Z-SZ3/libhdf5sz3.a + fi + + build-macos: + name: Build & Test (macOS) + runs-on: macos-latest + outputs: + digest: ${{ steps.calc.outputs.digest }} + steps: + - uses: actions/checkout@v5 + + - name: Install HDF5 + run: | + brew update + brew install hdf5 + + - name: Create build dir + run: cmake -E make_directory build + + - name: Configure + shell: bash + working-directory: build + run: | + cmake .. \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + shell: bash + working-directory: build + run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Run CTest + shell: bash + working-directory: build + run: ctest --output-on-failure --no-tests=error -C ${{ env.BUILD_TYPE }} + + - name: Download test data + shell: bash + run: | + curl -L -o input.dat ${{ env.TEST_URL }} + + - name: Run sz3 and check error + shell: bash + working-directory: build + run: | + ./tools/sz3/sz3 -f -i ../input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a \ + -o out.dat -z out.sz3 \ + | tee sz3.log + ../tools/test/checkMaxError.sh sz3.log ${{ env.TOL }} + + - name: Compute SHA256 of compressed + id: calc + shell: bash + working-directory: build + run: | + DIGEST=$(shasum -a 256 out.sz3 | awk '{print $1}') + [ -n "$DIGEST" ] || { echo "no digest to compare against Linux"; exit 1; } + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + + build-windows: + name: Build & Test (Windows) + runs-on: windows-latest + needs: build-linux + steps: + - uses: actions/checkout@v5 + + - name: Bootstrap vcpkg & install HDF5 + shell: pwsh + run: | + git clone https://github.com/microsoft/vcpkg.git C:\vcpkg + cd C:\vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg.exe install hdf5:x64-windows + echo "VCPKG_ROOT=C:\vcpkg" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + + - name: Create build dir + shell: pwsh + run: cmake -E make_directory build + + - name: Configure + shell: pwsh + working-directory: build + run: | + cmake .. ` + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} ` + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ` + -DVCPKG_TARGET_TRIPLET=x64-windows ` + -DBUILD_TESTING=ON ` + -DBUILD_H5Z_FILTER=ON + + - name: Build + shell: pwsh + working-directory: build + run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Run CTest + shell: pwsh + working-directory: build + run: ctest --output-on-failure --no-tests=error -C ${{ env.BUILD_TYPE }} + + - name: Build static libraries + shell: pwsh + run: | + # Static is what GROMACS builds on Windows; the H5Z tools link the static hdf5sz3. + cmake -S . -B build-static ` + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ` + -DVCPKG_TARGET_TRIPLET=x64-windows ` + -DBUILD_SHARED_LIBS=OFF ` + -DBUILD_H5Z_FILTER=ON + cmake --build build-static --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Download test data + shell: pwsh + run: Invoke-WebRequest -UseBasicParsing -Uri ${{ env.TEST_URL }} -OutFile input.dat + + - name: Run sz3 and check error + shell: pwsh + working-directory: build + run: | + .\tools\sz3\Release\sz3.exe -f -i ..\input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a ` + -o out.dat -z out.sz3 2>&1 | Tee-Object sz3.log + $match = Select-String -Pattern 'Max absolute error =\s*([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)' -Path sz3.log + if (-not $match) { + Write-Error "Could not parse Max absolute error!" + exit 1 + } + + $err = [double]$match.Matches[0].Groups[1].Value + Write-Host "Parsed error: $err" + + if ($err -gt [double]${{ env.TOL }}) { + Write-Error "Error $err exceeds tolerance ${{ env.TOL }}" + exit 1 + } + + + - name: Download Linux-compressed artifact + uses: actions/download-artifact@v4 + with: + name: temp-linux-out-sz3 + path: . + + - name: Run sz3 to decompress and check error + shell: pwsh + working-directory: build + run: | + .\tools\sz3\Release\sz3.exe -f -i ..\input.dat -z ..\out.sz3 -a -o out_recon.dat ` + 2>&1 | Tee-Object sz3.log + + $match = Select-String -Pattern 'Max absolute error =\s*([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)' -Path sz3.log + if (-not $match) { + Write-Error "Could not parse Max absolute error!" + exit 1 + } + + $err = [double]$match.Matches[0].Groups[1].Value + Write-Host "Parsed error: $err" + + if ($err -gt [double]${{ env.TOL }}) { + Write-Error "Error $err exceeds tolerance ${{ env.TOL }}" + exit 1 + } + + build-windows-mingw: + name: Build & Test (Windows MinGW) + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v5 + + - name: Setup MSYS2 with MinGW64 + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + mingw-w64-x86_64-gcc + mingw-w64-x86_64-libgomp + mingw-w64-x86_64-cmake + mingw-w64-x86_64-ninja + make + + # hdf5 comes in through a separate pacman call so it can carry --assume-installed. msys2's + # gcc-libgfortran pins gcc-libs to an exact build that the repository has already moved past, + # so fc-libs cannot resolve and pacman drops hdf5 from the transaction with only a warning. + # SZ3 asks for HDF5's C component alone, so the Fortran runtime is not ours to need. + - name: Install HDF5 without its Fortran runtime + run: | + pacman -S --noconfirm --needed \ + --assume-installed mingw-w64-x86_64-fc-libs \ + mingw-w64-x86_64-hdf5 + # pacman warns and returns 0 when it drops a package it could not resolve, which is how + # this went unnoticed in the first place. Ask for the file the build actually needs. + test -f /mingw64/include/hdf5.h + + - name: Create build dir + run: cmake -E make_directory build + + - name: Configure + working-directory: build + run: | + cmake .. \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + working-directory: build + run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Run CTest + working-directory: build + run: | + export PATH="$PWD/bin:$PWD/lib:$PATH" + ctest --output-on-failure --no-tests=error -C ${{ env.BUILD_TYPE }} + + - name: Download test data + run: | + curl -L -o input.dat ${{ env.TEST_URL }} + + - name: Run sz3 and check error + working-directory: build + run: | + ./tools/sz3/sz3.exe -f -i ../input.dat -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} -a \ + -o out.dat -z out.sz3 \ + | tee sz3.log + # Through bash: a Windows checkout carries no executable bit for the script to be run by. + bash ../tools/test/checkMaxError.sh sz3.log ${{ env.TOL }} + + compare: + name: Compare Digests (Linux vs macOS) + runs-on: ubuntu-latest + needs: [ build-linux, build-macos ] + steps: + - name: Check that Linux and macOS digests match + shell: bash + run: | + L=${{ needs.build-linux.outputs.digest }} + M=${{ needs.build-macos.outputs.digest }} + echo "linux: $L" + echo "macos: $M" + # Two digests that never arrived are equal to each other, so this job would report the + # platforms agreeing when neither had said anything. + if [ -z "$L" ] || [ -z "$M" ]; then + echo "::error ::A digest did not reach this job, so nothing was compared." + exit 1 + fi + if [[ "$L" != "$M" ]]; then + echo "::error ::Compressed outputs differ between Linux and macOS!" + exit 1 + fi + echo "Linux and macOS compressed outputs are identical." diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..efd82a34 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '45 21 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'cpp' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/gromacs.yml b/.github/workflows/gromacs.yml new file mode 100644 index 00000000..4abb44e0 --- /dev/null +++ b/.github/workflows/gromacs.yml @@ -0,0 +1,394 @@ +name: GROMACS + +# GROMACS is the consumer that an installed SZ3 has to satisfy, in two shapes: mainline finds it +# with find_package(SZ3) behind GMX_USE_SZ3, and the H5MD prototype vendors it and writes +# SZ3-compressed trajectories through it. +# +# Not on push. Measured at four cores this is around an hour of runner time across the matrix, and +# it depends on gitlab.com and a 70 MB download, so a broken third party would redden every commit. +# What it guards -- the shape of the installed export -- changes at the pace of a release. +# +# On a pull request it runs only when the pull request carries the `gromacs` label. schedule and +# workflow_dispatch are both read from the default branch, so neither can reach a workflow that is +# still on a branch: without this, the one change that most needs GROMACS run against it is the one +# change GROMACS cannot be run against. +on: + schedule: + - cron: "23 4 * * 1" + workflow_dispatch: + pull_request: + types: [labeled, synchronize, reopened] + +env: + GROMACS_URL: https://gitlab.com/gromacs/gromacs.git + # Mainline, on the branch that added GMX_USE_SZ3. Pinned by sha: on a moving branch a red run + # becomes a question about which GROMACS rather than an answer about SZ3. + GROMACS_MAIN_REF: 67345c60629773e67cd17f269b15a2cb303c6a9c + # The H5MD prototype. This is the branch on which SZ3-compressed H5MD output actually works. + GROMACS_PROTO_REF: fcc10d133b3411c9f3912b381f12a8e7515cf2dd + # MDAnalysisData's membrane peptide, 1001 frames of 18727 atoms: a real solvated membrane system, + # already named in tools/test/integration/datasets.json, and a .xtc/.tpr pair that gmx reads + # directly. 70 MB, against 1.34 GB for the Zenodo set, which buys nothing this does not. + TRAJ_URL: https://ndownloader.figshare.com/files/14993174 + TRAJ_SHA: f9bdfee4e1aa69ccfeef21cb74703202f6728f514543c4125382bd5250773eb7 + TOPOL_URL: https://ndownloader.figshare.com/files/14993171 + TOPOL_SHA: 677a3ae55e35c24f37f2610eafa92d19285d1774731d6ffb9a99dfde39b8c437 + # Nothing here runs a simulation: the cheapest GROMACS that still generates the h5md link line + # and builds the binaries these checks drive. + GMX_CHEAP: >- + -DCMAKE_BUILD_TYPE=Release + -DGMX_SIMD=None + -DGMX_FFT_LIBRARY=fftpack + -DGMX_GPU=OFF + -DGMX_OPENMP=OFF + -DGMX_BUILD_OWN_FFTW=OFF + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # GMX_USE_SZ3 in each of its four values, against an installed SZ3 that is present, absent, and + # present but unusable. Configure only, which is where the export decides what the consumer gets. + consumer: + name: GMX_USE_SZ3 ${{ matrix.name }} + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'gromacs') + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + # The OpenMPI wrappers in the environment carry no compiler of their own. + OMPI_CC: /usr/bin/gcc + OMPI_CXX: /usr/bin/g++ + strategy: + fail-fast: false + matrix: + include: + - name: serial + hdf5: hdf5=1.14.6 + bundled_zstd: "OFF" + gmx: "" + sz3: "" + - name: MPI + hdf5: hdf5=*=mpi_openmpi* + bundled_zstd: "OFF" + gmx: "-DGMX_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx" + sz3: "-DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx" + - name: bundled Zstd + hdf5: hdf5=1.14.6 + bundled_zstd: "ON" + gmx: "" + sz3: "" + steps: + - uses: actions/checkout@v5 + + - uses: mamba-org/setup-micromamba@v2 + with: + environment-name: h5 + create-args: >- + ${{ matrix.hdf5 }} + init-shell: bash + + - name: Install a system Zstd + run: | + sudo apt-get update + # libzstd-dev, not libzstd1: only the -dev package carries the libzstd.pc and the + # libzstd.so that the system-Zstd path needs at build and at find_package time. + sudo apt-get install -y libzstd-dev + + - name: Fetch GROMACS + run: | + git init -q gromacs + git -C gromacs remote add origin "$GROMACS_URL" + git -C gromacs fetch -q --depth 1 origin "$GROMACS_MAIN_REF" + git -C gromacs checkout -q FETCH_HEAD + git -C gromacs rev-parse HEAD + + - name: Build and install SZ3 + shell: bash -el {0} + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_H5Z_FILTER=ON \ + -DBUILD_TESTING=OFF \ + -DSZ3_USE_BUNDLED_ZSTD=${{ matrix.bundled_zstd }} \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" \ + ${{ matrix.sz3 }} + cmake --build build --parallel 4 + cmake --install build --prefix "$PWD/_prefix" + + - name: The installed SZ3 is the one GROMACS uses + shell: bash -el {0} + run: | + tools/test/gromacsConsumerMatrix.sh gromacs _m/found "$PWD/_prefix" "$CONDA_PREFIX" \ + found ${{ matrix.gmx }} + + - name: With no SZ3 to find, AUTO still configures + shell: bash -el {0} + run: | + tools/test/gromacsConsumerMatrix.sh gromacs _m/absent - "$CONDA_PREFIX" \ + absent ${{ matrix.gmx }} + + - name: An SZ3 built with OpenMP serves a GROMACS that finds none + shell: bash -el {0} + run: | + # SZ3Config.cmake adds OpenMP to SZ3::SZ3 only where the consumer finds it, and GROMACS links + # SZ3::hdf5sz3, which needs none. A decline leaving targets behind is consumeInstalledSZ3.sh's. + grep -qxE 'if\((ON|TRUE|1)\)' "$PWD/_prefix/lib/cmake/SZ3/SZ3Config.cmake" \ + || { echo "SZ3 was built without OpenMP, so this tests nothing"; exit 1; } + tools/test/gromacsConsumerMatrix.sh gromacs _m/noomp "$PWD/_prefix" "$CONDA_PREFIX" \ + found ${{ matrix.gmx }} -DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=ON + + # The link line is asserted at configure time above; this is the build that proves it, and the + # h5md unit tests that run against the library it linked. + h5md-tests: + name: h5md unit tests ${{ matrix.name }} + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'gromacs') + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + OMPI_CC: /usr/bin/gcc + OMPI_CXX: /usr/bin/g++ + strategy: + fail-fast: false + matrix: + include: + - name: serial + hdf5: hdf5=1.14.6 + gmx: "" + sz3: "" + - name: MPI + hdf5: hdf5=*=mpi_openmpi* + gmx: "-DGMX_MPI=ON -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx" + sz3: "-DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx" + steps: + - uses: actions/checkout@v5 + + - uses: mamba-org/setup-micromamba@v2 + with: + environment-name: h5 + create-args: >- + ${{ matrix.hdf5 }} + init-shell: bash + + - name: Install a system Zstd + run: | + sudo apt-get update + sudo apt-get install -y libzstd-dev + + - name: Fetch GROMACS + run: | + git init -q gromacs + git -C gromacs remote add origin "$GROMACS_URL" + git -C gromacs fetch -q --depth 1 origin "$GROMACS_MAIN_REF" + git -C gromacs checkout -q FETCH_HEAD + + - name: Build and install SZ3 + shell: bash -el {0} + run: | + cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_H5Z_FILTER=ON -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" ${{ matrix.sz3 }} + cmake --build build --parallel 4 + cmake --install build --prefix "$PWD/_prefix" + + - name: Build h5md-test against the installed SZ3 + shell: bash -el {0} + run: | + cmake -S gromacs -B gbuild $GMX_CHEAP \ + -DBUILD_TESTING=ON \ + -DGMX_USE_HDF5=ON \ + -DGMX_USE_SZ3=EXTERNAL \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX;$PWD/_prefix" \ + ${{ matrix.gmx }} + LINK=gbuild/src/gromacs/fileio/h5md/tests/CMakeFiles/h5md-test.dir/link.txt + tools/test/sz3LinkTokens.sh "$LINK" | tee sz3.link + if [ ! -s sz3.link ]; then + echo "h5md linked no SZ3 library at all"; exit 1 + fi + # Catches a bare -lhdf5sz3, which is the export failing to carry the unversioned target + # GROMACS asks for: it configures, and then the link below cannot find the library. + if grep -v "^$PWD/_prefix/" sz3.link; then + echo "h5md linked an SZ3 library that is not the installed one"; exit 1 + fi + echo "h5md links the installed SZ3 by path" + # GMX_OPENMP=OFF, and the shared filter passes no OpenMP on: none in h5md's compile or link. + FLAGS=gbuild/src/gromacs/fileio/h5md/CMakeFiles/h5md.dir/flags.make + [ -f "$FLAGS" ] || { echo "no $FLAGS to check"; exit 1; } + if grep -n -- '-fopenmp' "$FLAGS"; then + echo "h5md is compiled with the OpenMP GROMACS turned off"; exit 1 + fi + if grep -o '[^ ]*gomp[^ ]*' "$LINK"; then + echo "h5md-test links the OpenMP runtime GROMACS turned off"; exit 1 + fi + cmake --build gbuild --parallel 4 --target h5md-test + + - name: Run the h5md unit tests + shell: bash -el {0} + run: | + # pipefail, or tee would swallow a failing suite. + set -o pipefail + gbuild/bin/h5md-test | tee h5md.log + # gtest exits 0 for a run with no tests in it, so count them as well. + RAN=$(sed -n 's/^\[==========\] \([0-9]*\) tests from .* ran.*/\1/p' h5md.log | tail -1) + echo "tests run: ${RAN:-0}" + if [ "${RAN:-0}" -ge 100 ]; then + echo "the h5md suite ran" + else + echo "the h5md suite did not run"; exit 1 + fi + + # The part that is not "it builds": GROMACS writes a real trajectory through SZ3 and everything + # that reads it back is checked against the coordinates it started from. + roundtrip: + name: H5MD round trip + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'gromacs') + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v5 + + - uses: mamba-org/setup-micromamba@v2 + with: + # The prototype's own headers declare hsize_t as unsigned long long, which is what HDF5 + # was up to 1.12; against 1.14 or later its h5md sources do not compile at all. + environment-name: h5 + create-args: >- + hdf5=1.10.6 + numpy + init-shell: bash + + - name: Install a system Zstd + run: | + sudo apt-get update + sudo apt-get install -y libzstd-dev + + - name: Fetch the H5MD prototype + run: | + git init -q gromacs + git -C gromacs remote add origin "$GROMACS_URL" + git -C gromacs fetch -q --depth 1 origin "$GROMACS_PROTO_REF" + git -C gromacs checkout -q FETCH_HEAD + + - name: Put this SZ3 where the prototype vendors its own + run: | + # The prototype reaches SZ3 by add_subdirectory on src/external/SZ3-bio, which it ships at + # 3.1.8. The directory name is in its #include paths, so the name has to stay. + rm -rf gromacs/src/external/SZ3-bio + git archive HEAD --prefix=SZ3-bio/ | tar x -C gromacs/src/external + grep -m1 'project(SZ3' gromacs/src/external/SZ3-bio/CMakeLists.txt + python3 - <<'PY' + p = "gromacs/cmake/gmxManageHDF5.cmake" + s = open(p).read() + old = " set(BUILD_H5Z_FILTER ON)\n" + new = old + ( + " # SZ3 reads these itself. Without them BUILD_TESTING leaks in and SZ3\n" + " # fetches its own googletest, whose targets collide with GROMACS's.\n" + " set(BUILD_TESTING OFF)\n" + " set(BUILD_SZ3_BINARY OFF)\n" + " set(BUILD_MDZ OFF)\n" + " set(BUILD_PARAVIEW_PLUGIN OFF)\n") + assert s.count(old) == 1, "gmxManageHDF5.cmake no longer looks like this" + open(p, "w").write(s.replace(old, new)) + PY + + - name: Build gmx + shell: bash -el {0} + run: | + # trjconv is what writes .h5md, so this one needs the gmx driver rather than a test binary. + cmake -S gromacs -B gbuild $GMX_CHEAP \ + -DBUILD_TESTING=OFF \ + -DGMX_USE_HDF5=ON \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" + cmake --build gbuild --parallel 4 --target gmx + # The dependency names, not the directories they resolve in: every path here is under + # /home/runner/work/SZ3/SZ3, so a whole-line match answers a question about the checkout. + if ldd gbuild/bin/gmx | awk '{print $1}' | grep hdf5sz3; then + echo "gmx links the SZ3 filter library" + else + echo "gmx links no SZ3 filter library, so nothing below would be testing SZ3"; exit 1 + fi + + - name: Cache the trajectory + id: traj + uses: actions/cache@v4 + with: + path: data + key: mdanalysisdata-memb-pept-${{ env.TRAJ_SHA }} + + - name: Fetch the trajectory + if: steps.traj.outputs.cache-hit != 'true' + run: | + mkdir -p data + curl -sSL -o data/memb_pept.xtc "$TRAJ_URL" + curl -sSL -o data/memb_pept.tpr "$TOPOL_URL" + + - name: Check the trajectory + run: | + # Also on a cache hit: a cache is not a checksum. + echo "$TRAJ_SHA data/memb_pept.xtc" | sha256sum -c - + echo "$TOPOL_SHA data/memb_pept.tpr" | sha256sum -c - + + - name: Round trip at three error bounds + shell: bash -el {0} + run: | + export PYTHON=$CONDA_PREFIX/bin/python + # The filter has to identify itself as the SZ3 in this tree, not as the 3.1.8 the + # prototype vendored. + export SZ3_EXPECT_VERSION=$(sed -n 's/^project(SZ3 VERSION \([0-9.]*\)).*/\1/p' CMakeLists.txt) + # sed exits 0 when it matches nothing, and an empty pin leaves the assertion looking for + # "H5Z-SZ3-", which the vendored 3.1.8 satisfies as readily as this tree does. + [ -n "$SZ3_EXPECT_VERSION" ] \ + || { echo "could not read the version out of project(SZ3 VERSION ...)"; exit 1; } + echo "expecting filter version $SZ3_EXPECT_VERSION" + # -ndec 1, 2 and 3 ask SZ3 for 0.05, 0.005 and 0.0005 nm. One bound says the path works; + # three say the bound is the thing that moves. + for ndec in 1 2 3; do + tools/test/gromacsH5mdRoundtrip.sh gbuild/bin/gmx gbuild/lib "$CONDA_PREFIX/bin" \ + data/memb_pept.xtc data/memb_pept.tpr "_rt/n$ndec" $ndec + # Each round holds about a gigabyte of trajectory; the runner has 14. + rm -rf "_rt/n$ndec" + done + + - name: Registering the filter it links lets an application skip HDF5_PLUGIN_PATH + shell: bash -el {0} + run: | + mkdir -p _init + cd _init + run_it() { + echo 0 | env -u HDF5_PLUGIN_PATH ../gbuild/bin/gmx trjconv \ + -f ../data/memb_pept.xtc -s ../data/memb_pept.tpr -o "$1" -ndec 2 -e 50 \ + > "$1.log" 2>&1 + } + # As shipped, the prototype registers the filter through H5Pset_filter alone, so it needs + # the plugin on HDF5_PLUGIN_PATH even though it links the library. + if run_it before.h5md; then + echo "expected the prototype to fail without HDF5_PLUGIN_PATH, and it did not"; exit 1 + fi + if grep -m1 "SZ3 compression filter" before.h5md.log; then + echo "and it failed for the documented reason" + else + echo "it failed for some other reason"; tail -20 before.h5md.log; exit 1 + fi + cd .. + python3 - <<'PY' + p = "gromacs/src/gromacs/fileio/h5md_low_level_util.cpp" + s = open(p).read() + old = "void registerSz3FilterImplicitly()\n{\n" + new = old + " H5Zregister(H5PLget_plugin_info());\n" + assert s.count(old) == 1, "registerSz3FilterImplicitly no longer looks like this" + open(p, "w").write(s.replace(old, new)) + PY + cmake --build gbuild --parallel 4 --target gmx + cd _init + if run_it after.h5md; then + echo "the same call now works with no HDF5_PLUGIN_PATH set" + else + tail -20 after.h5md.log; exit 1 + fi + # And what it wrote without the plugin path is SZ3-compressed, not a silent fallback. + HDF5_PLUGIN_PATH=/nonexistent "$CONDA_PREFIX/bin/h5dump" -pH \ + -d /particles/system/position/value after.h5md > after.layout + if grep -q "FILTER_ID 32024" after.layout; then + grep -E 'FILTER_ID|COMMENT' after.layout + else + echo "the file it wrote carries no SZ3 filter"; cat after.layout; exit 1 + fi diff --git a/.github/workflows/hdf5.yml b/.github/workflows/hdf5.yml new file mode 100644 index 00000000..452efdaa --- /dev/null +++ b/.github/workflows/hdf5.yml @@ -0,0 +1,189 @@ +name: HDF5 + +on: [ push ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BUILD_TYPE: Release + +# The filter against more than one HDF5 at a time: several versions, a parallel one, and two +# installed side by side. What a build with a single HDF5 on the machine cannot see is whether the +# export hands a consumer paths belonging to the HDF5 we built against instead of theirs. +jobs: + versions: + name: HDF5 ${{ matrix.hdf5 }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + hdf5: ["1.10.6", "1.14.6", "2.2.0"] + steps: + - uses: actions/checkout@v5 + + - uses: mamba-org/setup-micromamba@v2 + with: + environment-name: h5 + create-args: >- + hdf5=${{ matrix.hdf5 }} + zstd + init-shell: bash + + # One step each, because the matrix leg's name says only which HDF5 and these fail for + # unrelated reasons: a build failure is the filter's API drifting, a ctest failure is its + # behaviour, and a consumer failure is the export handing out this job's HDF5 paths. + - name: Configure + shell: bash -el {0} + run: | + # The filter's API moved between these: H5Zregister's class struct, the H5PL entry points + # and h5repack's own behaviour all differ across 1.10, 1.14 and 2.x. + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=ON \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" + + - name: Build + shell: bash -el {0} + run: cmake --build build --parallel 4 + + - name: Run CTest + shell: bash -el {0} + working-directory: build + run: ctest --output-on-failure --no-tests=error -C ${{ env.BUILD_TYPE }} + + - name: Install it for a consumer to find + shell: bash -el {0} + run: cmake --install build --prefix "$PWD/_prefix" + + - name: Every shape a consumer builds in + shell: bash -el {0} + run: tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" "$CONDA_PREFIX" + + - name: Every way an application reaches the filter + shell: bash -el {0} + run: tools/H5Z-SZ3/test/filterSuite.sh "$PWD/_prefix" "$CONDA_PREFIX/bin" + + coexist: + name: Two HDF5s on one machine + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: mamba-org/setup-micromamba@v2 + with: + environment-name: h5build + create-args: >- + hdf5=1.14.6 + zstd + init-shell: bash + + - name: Install a second HDF5 the build will not see + shell: bash -el {0} + run: micromamba create -y -n h5use -c conda-forge hdf5=2.2.0 + + - name: A consumer compiles against its own HDF5, not ours + shell: bash -el {0} + run: | + # The export must name no include directory belonging to the HDF5 SZ3 was built against. + # Unguarded, it did, and a consumer who found a different HDF5 compiled against ours. + USE=$MAMBA_ROOT_PREFIX/envs/h5use + cmake -S . -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DBUILD_H5Z_FILTER=ON \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" + cmake --build build --parallel 4 + cmake --install build --prefix "$PWD/_prefix" + mkdir -p _c + cat > _c/CMakeLists.txt <<'EOF' + cmake_minimum_required(VERSION 3.18) + project(c CXX) + find_package(SZ3 REQUIRED) + add_executable(app main.cpp) + target_link_libraries(app PRIVATE SZ3 hdf5sz3) + EOF + cat > _c/main.cpp <<'EOF' + #include + #include + int main() { + printf("%d.%d.%d\n", H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE); + return 0; + } + EOF + cmake -S _c -B _c/build -DCMAKE_PREFIX_PATH="$PWD/_prefix;$USE" + cmake --build _c/build --parallel 4 + SAW=$(./_c/build/app) + echo "consumer compiled against HDF5 $SAW; SZ3 was built against 1.14.6" + if [ "$SAW" != "2.2.0" ]; then + echo "the consumer got the HDF5 SZ3 was built against, not the one it asked for"; exit 1 + fi + + parallel: + name: Parallel HDF5 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install HDF5 and Zstd + run: | + sudo apt-get update + # libzstd-dev, not libzstd1: only the -dev package carries the libzstd.pc that sends this + # job down the system-Zstd path. + sudo apt-get install -y libhdf5-dev libzstd-dev + + - name: Create build dir + run: cmake -E make_directory build + + - name: Configure + shell: bash + working-directory: build + run: | + # No BUILD_TESTING: nothing in this job runs ctest, and it would install googletest into + # the prefix the consumer below finds SZ3 in. + cmake .. \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + shell: bash + working-directory: build + run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Install it for the consumer to find + shell: bash + run: cmake --install build --prefix "$PWD/_prefix" + + - name: A parallel-HDF5 consumer needs no MPI search of ours + shell: bash + run: | + # SZ3Config.cmake resolves no MPI: the filter calls none, and a parallel HDF5 carries the + # MPI it was built against. This is the case that proves it -- if it did not hold, the + # consumer would end up with a second, independently chosen MPI. + sudo apt-get install -y libhdf5-openmpi-dev openmpi-bin + HDF5_MPI_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/openmpi + mkdir -p _mpi + cat > _mpi/CMakeLists.txt <<'EOF' + cmake_minimum_required(VERSION 3.18) + project(mpiconsumer C) + find_package(SZ3 REQUIRED) + add_executable(mapp main.c) + target_link_libraries(mapp PRIVATE SZ3::hdf5sz3) + EOF + cat > _mpi/main.c <<'EOF' + #include + #include + int main(void) { + htri_t avail = H5Zfilter_avail(32024); + printf("filter avail=%d\n", (int)avail); + return avail > 0 ? 0 : 1; + } + EOF + cmake -S _mpi -B _mpi/build \ + -DCMAKE_PREFIX_PATH="$PWD/_prefix;$HDF5_MPI_ROOT" \ + -DCMAKE_C_COMPILER=mpicc + cmake --build _mpi/build --parallel 4 + HDF5_PLUGIN_PATH="$PWD/_prefix/lib/plugin" ./_mpi/build/mapp + ldd _mpi/build/mapp | grep -E 'libmpi|libhdf5' | tee _mpi.log + if [ "$(grep -c 'libmpi' _mpi.log)" -gt 1 ]; then + echo "more than one MPI in the consumer's binary"; cat _mpi.log; exit 1 + fi diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml new file mode 100644 index 00000000..cbf4ce46 --- /dev/null +++ b/.github/workflows/integration_test.yml @@ -0,0 +1,111 @@ +name: Integration Test + +on: + pull_request: + branches: + - master + +# One run per pull request. A newer push supersedes the one in flight, which otherwise queues a +# second full matrix and evicts the long jobs of the first. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + # A dataset entry may name the fields to run, so one too slow to be a single job is spread + # over several. hacc took 2h09 as one job, scale-letkf 1h00 and exaalt-helium 46m; the rest + # were already under half an hour. Split to keep every job near 35 minutes. + dataset: + - exaalt-small + - cesm-atm + - miranda + - hurricane + - exaalt-copper + - "hacc:vx.f32" + - "hacc:vz.f32" + - "hacc:vy.f32" + - "hacc:zz.f32" + - "hacc:xx.f32" + - "hacc:yy.f32" + - "exaalt-helium:dataset1-7852x1037.x.f32.dat,dataset1-7852x1037.y.f32.dat,dataset1-7852x1037.z.f32.dat" + - "exaalt-helium:dataset2-2338x106711.x.f32.dat,dataset2-2338x106711.y.f32.dat,dataset2-2338x106711.z.f32.dat" + - "scale-letkf:T-98x1200x1200.f32,PRES-98x1200x1200.f32,U-98x1200x1200.f32,V-98x1200x1200.f32,QC-98x1200x1200.f32,QS-98x1200x1200.f32" + - "scale-letkf:QG-98x1200x1200.f32,QV-98x1200x1200.f32,W-98x1200x1200.f32,QI-98x1200x1200.f32,RH-98x1200x1200.f32,QR-98x1200x1200.f32" + # Molecular dynamics trajectories, which arrive as {frames, atoms, xyz}. Nothing above + # has three as its last dimension, so ALGO_BIOMD only ever sees the + # layout it was written for through this. + - md-peg-1chain + - md-ifabp-water + - md-adk-equilibrium + - md-cg-fiber + - md-membrane-peptide + - md-nhaa-equilibrium + - md-yiip-equilibrium + fail-fast: false + name: Test ${{ matrix.dataset }} + + steps: + + - name: Disk space report before modification + shell: bash + run: | + echo "Memory and swap:" + sudo free -h + echo + sudo swapon --show + echo + + echo "Available storage:" + sudo df -h + echo + + - name: Maximize build disk space + shell: bash + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + sudo docker builder prune -a + + - name: Disk space report after modification + shell: bash + run: | + echo "Memory and swap:" + sudo free -h + echo + sudo swapon --show + echo + + echo "Available storage:" + sudo df -h + + - uses: actions/checkout@v5 + + - name: Set up Python 3 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev hdf5-tools + python -m pip install --upgrade pip + pip install numpy h5py requests --no-binary h5py + # Reads the DCD, XTC and NetCDF trajectories the md-* datasets are published as. + pip install MDAnalysis + + - name: Run integration test + run: | + python tools/test/integration/integration_test_driver.py tools/test/integration/datasets.json ${{ matrix.dataset }} + + - name: Upload compression metrics + if: always() + uses: actions/upload-artifact@v4 + with: + name: metrics-${{ strategy.job-index }} + path: integration_metrics.csv + if-no-files-found: ignore diff --git a/.github/workflows/libpressio.yml b/.github/workflows/libpressio.yml new file mode 100644 index 00000000..46e5d5a0 --- /dev/null +++ b/.github/workflows/libpressio.yml @@ -0,0 +1,79 @@ +name: libpressio + +on: [ push ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BUILD_TYPE: Release + # Both pinned at their release commits: on a moving branch a red run becomes a question about + # which libpressio rather than an answer about SZ3. + LIBPRESSIO_REF: 868a3a70d6ebf55ad67509fbca03bdd0bc1bc246 # v1.0.7, 2025-09-18 + STD_COMPAT_REF: 514383ce8816b7bcb800fbe3ce842c1e838105e2 # v0.0.22, 2025-08-07 + +# libpressio reaches SZ3 the way a packaged consumer does -- find_package(SZ3), SZ3::SZ3 -- and is +# the only thing in CI that compiles anyone else's code against our API. libpressio has no CI of +# its own to catch a break at their end. On push, not gromacs.yml's schedule: this takes minutes. +jobs: + build: + name: libpressio builds against an installed SZ3 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install Zstd + run: | + sudo apt-get update + # Both sides need it: it sends SZ3 down the system-Zstd path, and libpressio resolves + # libzstd through pkg-config on the branch that its SZ3 plugin is on. + sudo apt-get install -y libzstd-dev + + - name: Install SZ3 for libpressio to find + shell: bash + run: | + # No binaries, no filter, no BUILD_TESTING: libpressio links SZ3::SZ3 and reaches nothing + # else in the export. + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_SZ3_BINARY=OFF + cmake --build build --parallel 4 + cmake --install build --prefix "$PWD/_prefix" + + - name: Install std_compat, the one dependency libpressio always needs + shell: bash + run: | + # Header-only and seconds to build, but packaged under that name nowhere a runner can + # install it from -- not apt, not conda-forge. Into the same prefix as SZ3, which is the + # shape a consumer on a conda or Spack view sees. + git clone -q https://github.com/robertu94/std_compat.git _std_compat + git -C _std_compat checkout -q ${{ env.STD_COMPAT_REF }} + # CMAKE_INSTALL_PREFIX at configure time, not --prefix at install time: its .pc files have + # an absolute DESTINATION built from the prefix, which --prefix cannot move, so an install + # driven that way writes to /usr/local/share/pkgconfig and stops there. + cmake -S _std_compat -B _std_compat/build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DCMAKE_INSTALL_PREFIX="$PWD/_prefix" \ + -DBUILD_TESTING=OFF + cmake --build _std_compat/build --parallel 4 + cmake --install _std_compat/build + + - name: libpressio compiles and links against this SZ3 + shell: bash + run: | + git clone -q https://github.com/robertu94/libpressio.git _libpressio + git -C _libpressio checkout -q ${{ env.LIBPRESSIO_REF }} + cmake -S _libpressio -B _libpressio/build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_TESTING=OFF \ + -DLIBPRESSIO_HAS_SZ3=ON \ + -DCMAKE_PREFIX_PATH="$PWD/_prefix" + # The configure is not the check. With SZ3/api/sz.hpp replaced by a bare #error it still + # exits 0, because find_package(SZ3) reads the export and compiles nothing. The plugin's + # own translation unit is the only place the API surface is looked at. + cmake --build _libpressio/build --parallel 4 + # Named, not assumed: with LIBPRESSIO_HAS_SZ3 off this builds just as happily, and the + # step would then be reporting on a library with no SZ3 in it. + nm -C _libpressio/build/liblibpressio.so | grep 'sz3_ns::registration' > /dev/null \ + || { echo "the library carries no sz3 plugin, so nothing here tested SZ3"; exit 1; } diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml new file mode 100644 index 00000000..08a53871 --- /dev/null +++ b/.github/workflows/packaging.yml @@ -0,0 +1,369 @@ +name: Packaging + +on: [ push ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BUILD_TYPE: Release + DIMS: "128 8 8" + MODE: ABS + TOL: 1 + +# What an installed SZ3 hands someone else: the export, the consumer shapes it has to support, and +# the promises the bundled Zstd makes. None of this is visible to a build that only ever runs from +# its own build tree, which is how every test in this repository reached SZ3 before. +jobs: + linux: + name: Install and consume (Linux) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Install HDF5 and Zstd + run: | + sudo apt-get update + # libzstd-dev, not libzstd1: only the -dev package carries the libzstd.pc that sends this + # job down the system-Zstd path. hdf5-tools because this job runs filterSuite.sh + # directly rather than through ctest, and the script drives h5repack, h5dump and h5ls. + sudo apt-get install -y libhdf5-dev hdf5-tools libzstd-dev + + - name: Configure + shell: bash + run: | + # No BUILD_TESTING: nothing here runs ctest, and it makes FetchContent pull googletest, + # whose own install rules put libgtest, libgmock and their headers into the prefix this + # job then inspects as an installed SZ3. + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + shell: bash + run: cmake --build build --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Check this build took the system-Zstd path + shell: bash + run: | + if ! ldd build/tools/sz3/sz3 | grep -q 'libzstd\.so'; then + echo "sz3 links no system libzstd, so this job is not testing the system path" + ldd build/tools/sz3/sz3; exit 1 + fi + if find build -name 'libsz3_zstd.*' | grep -q .; then + echo "the bundled Zstd was built as well, so which one is linked is not clear"; exit 1 + fi + # Named, not globbed at the grep: CMake picks the directory under Export/ and an + # unmatched glob reaches grep as a literal, which exits 2 and reads here as "clean". + EXPORT=$(echo build/CMakeFiles/Export/*/SZ3Targets.cmake) + [ -f "$EXPORT" ] || { echo "no generated export to check at $EXPORT"; exit 1; } + if grep -q 'SZ3::zstd' "$EXPORT"; then + echo "the export names SZ3::zstd, so this build did not take the system path"; exit 1 + fi + + - name: Install it for a consumer to find + shell: bash + run: cmake --install build --prefix "$PWD/_prefix" + + # Spack's sz3 package runs this installed binary as its install test. + - name: The installed sz3_smoke_test runs + shell: bash + run: ./_prefix/bin/sz3_smoke_test + + - name: Every shape a consumer builds in + shell: bash + run: tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" + + - name: Every way an application reaches the filter + shell: bash + run: tools/H5Z-SZ3/test/filterSuite.sh "$PWD/_prefix" + + # BUILD_H5Z_FILTER is off by default, so a packaged SZ3 is likely to come without the filter; + # find_package(SZ3 COMPONENTS hdf5sz3) must then decline and leave no targets behind. + - name: An SZ3 installed without the filter + shell: bash + run: | + cmake -S . -B build-nofilter -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} + cmake --build build-nofilter --config ${{ env.BUILD_TYPE }} --parallel 4 + cmake --install build-nofilter --prefix "$PWD/_prefix_nofilter" + tools/test/consumeInstalledSZ3.sh "$PWD/_prefix_nofilter" + + - name: The installed filter resolves every library it needs + shell: bash + run: | + ldd _prefix/lib/libhdf5sz3.so | tee _ldd.log + # Not `! grep ...`: set -e exempts a !-inverted command, so it would never fail the step. + if grep -q 'not found' _ldd.log; then echo "a NEEDED library does not resolve"; exit 1; fi + + - name: The export must not carry this machine's Zstd + shell: bash + run: | + # Listed before it is grepped, for the reason the generated-export check above gives. + # One file per configuration, so the variable is split on purpose. + EXPORTS=$(ls _prefix/lib/cmake/SZ3/SZ3Targets*.cmake 2> /dev/null) + [ -n "$EXPORTS" ] || { echo "no installed export to check under _prefix/lib/cmake/SZ3"; exit 1; } + # SZ3Config.cmake is deliberately not grepped: its find_library runs on the consumer's machine. + if grep -En 'libzstd|PkgConfig::ZSTD' $EXPORTS; then + echo "the installed export names this machine's Zstd"; exit 1 + fi + + - name: A consumer still builds with the build tree gone + shell: bash + run: | + # An export that reaches back into the tree it was generated in works on this machine and + # nowhere else. Moved aside rather than deleted, so a red run still has the build to look at. + mv build build.aside + tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" + mv build.aside build + + - name: A packager can delete tools/zstd and still build against the system Zstd + shell: bash + run: | + # Debian and Fedora forbid embedded code copies, so the vendored tree has to be deletable. + # git worktree, so the deletion cannot escape. + git worktree add _nozstd HEAD + rm -rf _nozstd/tools/zstd + cmake -S _nozstd -B _nozstd/build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} + cmake --build _nozstd/build --parallel 4 + EXPORT=$(echo _nozstd/build/CMakeFiles/Export/*/SZ3Targets.cmake) + [ -f "$EXPORT" ] || { echo "no generated export to check at $EXPORT"; exit 1; } + if grep -q 'SZ3::zstd' "$EXPORT"; then + echo "with tools/zstd gone this did not take the system path"; exit 1 + fi + _nozstd/build/tools/sz3/sz3 -f -i tools/sz3/testfloat_8_8_128.dat -z _n.sz3 \ + -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} + _nozstd/build/tools/sz3/sz3 -f -s _n.sz3 -o _n.dat -3 ${{ env.DIMS }} + if [ "$(wc -c < _n.dat)" != "$(wc -c < tools/sz3/testfloat_8_8_128.dat)" ]; then + echo "the system Zstd did not reproduce the input length"; exit 1 + fi + + - name: Build and install with the bundled Zstd + shell: bash + run: | + cmake -S . -B _bundled -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DSZ3_USE_BUNDLED_ZSTD=ON -DBUILD_H5Z_FILTER=ON + cmake --build _bundled --parallel 4 + cmake --install _bundled --prefix "$PWD/_bprefix" + + - name: The bundled Zstd round-trips the smoke dataset + shell: bash + run: | + _bundled/tools/sz3/sz3 -f -i tools/sz3/testfloat_8_8_128.dat -z _b.sz3 \ + -3 ${{ env.DIMS }} -M ${{ env.MODE }} ${{ env.TOL }} + _bundled/tools/sz3/sz3 -f -s _b.sz3 -o _b.dat -3 ${{ env.DIMS }} + if [ "$(wc -c < _b.dat)" != "$(wc -c < tools/sz3/testfloat_8_8_128.dat)" ]; then + echo "the bundled Zstd did not reproduce the input length"; exit 1 + fi + + - name: The bundled Zstd stays inside SZ3 + shell: bash + run: tools/test/bundledZstdIsolation.sh _bundled "$PWD/_bprefix" + + - name: The serialized format must not change without SZ3_DATA_VERSION + shell: bash + run: | + # An unversioned format change produces files an older build accepts and then misreads. + # The _bundled tree: the bytes depend on which Zstd compressed them, and only it is pinned. + VER=$(sed -n 's/^#define SZ3_DATA_VER "\(.*\)"/\1/p' _bundled/include/SZ3/version.hpp) + if [ -z "$VER" ]; then echo "could not read SZ3_DATA_VER from the generated header"; exit 1; fi + ./_bundled/tools/sz3/sz3 -f -i tools/sz3/testfloat_8_8_128.dat -z fmt.sz3 \ + -3 8 8 128 -M ABS 1e-3 > /dev/null + GOT=$(sha256sum fmt.sz3 | awk '{print $1}') + WANT=$(awk -v v="$VER" '$1 == v { print $2 }' tools/test/data_format_digest.txt) + echo "SZ3_DATA_VERSION=$VER" + echo " recorded: ${WANT:-}" + echo " produced: $GOT" + if [ -z "$WANT" ]; then + echo "::error ::SZ3_DATA_VERSION $VER has no row in tools/test/data_format_digest.txt. Add: $VER $GOT" + exit 1 + fi + if [ "$WANT" != "$GOT" ]; then + echo "::error ::The bytes this version produces have changed. If the serialized format changed, bump SZ3_DATA_VERSION in CMakeLists.txt and add a row for the new version. If the bundled Zstd was re-vendored, replace this row with: $VER $GOT" + exit 1 + fi + + macos: + name: Install and consume (macOS) + runs-on: macos-latest + steps: + - uses: actions/checkout@v5 + + - name: Install HDF5 + run: | + brew update + brew install hdf5 + + - name: Create build dir + run: cmake -E make_directory build + + - name: Configure + shell: bash + working-directory: build + run: | + cmake .. \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_TESTING=ON \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + shell: bash + working-directory: build + run: cmake --build . --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Install it for a consumer to find + shell: bash + run: cmake --install build --prefix "$PWD/_prefix" + + # Spack's sz3 package runs this installed binary as its install test. + - name: The installed sz3_smoke_test runs + shell: bash + run: ./_prefix/bin/sz3_smoke_test + + - name: Every shape a consumer builds in + shell: bash + run: tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" + + - name: The installed filter loads by the name recorded in it + shell: bash + run: | + # Mach-O resolves libraries by install name rather than RUNPATH, so an export that works + # on Linux can still fail here. + otool -L _prefix/lib/libhdf5sz3.dylib + # The load is the assertion, not the otool output. Line 2 of `otool -L` is this library's + # own LC_ID_DYLIB -- always @rpath/libhdf5sz3.dylib -- so a grep for '@rpath/libhdf5' + # matches whatever HDF5 does, and the presence of *an* LC_RPATH says nothing about + # whether it holds libhdf5. + printf '#include \n#include \nint main(int c,char**v){(void)c;void*h=dlopen(v[1],RTLD_NOW);if(!h){printf("%%s\\n",dlerror());return 1;}return 0;}\n' > _load.c + cc -o _load _load.c + ./_load "$PWD/_prefix/lib/libhdf5sz3.dylib" + ./_load "$PWD/_prefix/lib/plugin/libhdf5sz3.dylib" + + - name: Every way an application reaches the filter + shell: bash + run: tools/H5Z-SZ3/test/filterSuite.sh "$PWD/_prefix" + + windows-msvc: + name: Install and consume (Windows MSVC) + runs-on: windows-latest + steps: + - uses: actions/checkout@v5 + + # As in cmake.yml's Windows job: vcpkg is where an MSVC build of this repository gets HDF5. + - name: Bootstrap vcpkg & install HDF5 + shell: pwsh + run: | + git clone https://github.com/microsoft/vcpkg.git C:\vcpkg + cd C:\vcpkg + .\bootstrap-vcpkg.bat + .\vcpkg.exe install hdf5:x64-windows + + - name: Configure + shell: bash + run: | + # No BUILD_TESTING, for the reason the Linux job gives. BUILD_MDZ because no other job in + # any workflow builds tools/mdz at all, and its install rule is one of the two a + # multi-config generator breaks -- the other being tools/sz3, which this job installs too. + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-windows \ + -DBUILD_H5Z_FILTER=ON \ + -DBUILD_MDZ=ON + + - name: Build + shell: bash + run: cmake --build build --config ${{ env.BUILD_TYPE }} --parallel 4 + + - name: Install it for a consumer to find + shell: bash + # --config here as well as on the build: the Visual Studio generator is multi-config, and an + # install that names no configuration looks for Release whatever was built. + run: cmake --install build --config ${{ env.BUILD_TYPE }} --prefix "$PWD/_prefix" + + - name: Every binary the install rules promise + shell: bash + run: | + # An install rule that points one directory too high, or drops the .exe, is how this got + # here: `cmake --install` failed outright and nothing after it reached the prefix. + for exe in sz3 sz3_smoke_test mdz mdz_smoke_test; do + [ -f "_prefix/bin/$exe.exe" ] || { echo "no $exe.exe in the install prefix"; exit 1; } + done + ./_prefix/bin/sz3_smoke_test.exe + ./_prefix/bin/mdz_smoke_test.exe + # Named here as well as inside consumeInstalledSZ3.sh, because Linux asserts the filter + # with ldd and macOS with a dlopen probe and this job had neither: a filter that failed to + # install at all left the script with nothing to detect and nothing to consume. + # No lib prefix under MSVC, and the import library stays out of the plugin directory. + for f in bin/hdf5sz3.dll lib/hdf5sz3.lib lib/plugin/hdf5sz3.dll \ + lib/cmake/SZ3/HDF5SZ3.cmake include/hdf5_sz3/H5Z_SZ3.hpp; do + [ -f "_prefix/$f" ] || { echo "no $f in the install prefix"; exit 1; } + done + + - name: Every shape a consumer builds in + shell: bash + run: | + # The consumer links HDF5's import library and Windows resolves the matching DLL through + # PATH, which no RPATH in an install tree can stand in for. + export PATH="/c/vcpkg/installed/x64-windows/bin:$PATH" + tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" /c/vcpkg/installed/x64-windows + + windows-mingw: + name: Install and consume (Windows MinGW) + runs-on: windows-latest + defaults: + run: + shell: msys2 {0} + steps: + - uses: actions/checkout@v5 + + - name: Setup MSYS2 with MinGW64 + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + mingw-w64-x86_64-gcc + mingw-w64-x86_64-libgomp + mingw-w64-x86_64-cmake + mingw-w64-x86_64-ninja + make + + # hdf5 comes in through a separate pacman call so it can carry --assume-installed. msys2's + # gcc-libgfortran pins gcc-libs to an exact build that the repository has already moved past, + # so fc-libs cannot resolve and pacman drops hdf5 from the transaction with only a warning. + # SZ3 asks for HDF5's C component alone, so the Fortran runtime is not ours to need. + - name: Install HDF5 without its Fortran runtime + run: | + pacman -S --noconfirm --needed \ + --assume-installed mingw-w64-x86_64-fc-libs \ + mingw-w64-x86_64-hdf5 + # pacman warns and returns 0 when it drops a package it could not resolve, which is how + # this went unnoticed in the first place. Ask for the file the build actually needs. + test -f /mingw64/include/hdf5.h + + - name: Configure + run: | + # No BUILD_TESTING, for the reason the Linux job gives. + cmake -S . -B build \ + -G "Ninja" \ + -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ + -DBUILD_H5Z_FILTER=ON + + - name: Build + run: cmake --build build --parallel 4 + + - name: Install it for a consumer to find + run: cmake --install build --prefix "$PWD/_prefix" + + - name: Every binary the install rules promise + run: | + # As in the MSVC job. MinGW keeps the lib prefix MSVC drops, and names the import + # library .dll.a, so the two Windows jobs cannot share one list. + for f in bin/libhdf5sz3.dll lib/libhdf5sz3.dll.a lib/plugin/libhdf5sz3.dll \ + lib/cmake/SZ3/HDF5SZ3.cmake include/hdf5_sz3/H5Z_SZ3.hpp; do + [ -f "_prefix/$f" ] || { echo "no $f in the install prefix"; exit 1; } + done + + - name: Every shape a consumer builds in + run: tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..07817876 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,17 @@ +name: Close stale issues + +on: + schedule: + - cron: '0 0 * * *' # runs daily at midnight + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v10 + with: + days-before-stale: 180 + days-before-close: 30 + stale-issue-message: 'This issue has been automatically marked as stale due to inactivity.' + close-issue-message: 'Closing this issue due to prolonged inactivity.' + exempt-issue-labels: 'keep-open' diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..5a8e409e --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +test.dat +compressed.out +perf.data* +# Anchored to the repo root, and glob so build-release and friends are covered too +/build* +debug +cmake-build-debug +cmake-build-release +.idea +install +.DS_Store +.vscode +.cache +.claude +tmp +docs/site/html +sz3_install +sz3_build +/test + +# Python bytecode +__pycache__/ +*.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..ba19dafc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,336 @@ +# SZ3 version history + +Oldest version first. Each entry says whether the version has a git tag and a GitHub release, and which data formats it can decompress when that changed. A version that was current for less than two weeks has no entry of its own; its changes are listed under the next version that has one. + +## 3.0.0 (2021-01-22) + +No tag and no GitHub release. This is commit `fb76989`, "first release of sz3". + +- First release of SZ3, the C++ version of SZ. A compressor is put together from separate modules: predictor, quantizer, encoder and lossless stage. +- Header-only C++17 library. `sz_demo` compresses float data with Lorenzo and regression prediction, Huffman coding and Zstd. +- Zstd had to be pointed to by editing `ZSTD_LIBS` and `ZSTD_INCLUDES` in `CMakeLists.txt`. + +## 3.0.1 (2021-04-15) + +Tag `v3.0.1`. The GitHub release has no notes. + +**New** +- Zstd is found through pkg-config. If it is not found, a bundled copy of Zstd 1.3.5 is built. +- Arithmetic and bypass encoders, a pre-processor stage (transpose, filter), and a truncation compressor with its example `sz_truncate`. +- `sz_fast`, a compressor built on `SZMetaFrontend`, which follows the SZauto design. + +**Changes** +- CMake 3.13 or newer is required. + +**Fixes** +- Faster Huffman and arithmetic decoding. + +## 3.0.2 (2021-06-01) + +Tag `v3.0.2`. The GitHub release has no notes. + +**New** +- Point-wise relative error bound, through the separate example `sz_pw`. +- The Zstd level can be set when constructing `Lossless_zstd`. +- The truncation compressor can keep a chosen number of bytes per value. + +## 3.1.3 (2022-02-04) + +Tag `v3.1.3`. The GitHub release is titled "V3.1.3". This entry also covers 3.1.0 (tag `v3.1.0`, 2022-01-21), 3.1.1 (tag `v3.1.1`, 2022-01-26) and 3.1.2 (tag `v3.1.2`, 2022-02-02). Each was current for less than two weeks, and each has a GitHub release. + +**New** +- Interpolation-based compression (the ICDE'21 dynamic spline interpolation). The default algorithm, `ALGO_INTERP_LORENZO`, chooses between interpolation and Lorenzo prediction and tunes its settings automatically. `ALGO_INTERP` and `ALGO_LORENZO_REG` pick one predictor family. +- A public API in `include/SZ3/api/sz.hpp`: `SZ_compress(conf, data, outSize)` and `SZ_decompress(conf, cmpData, cmpSize)`, configured with `SZ::Config`. `SZ_decompress(conf, cmpData, cmpSize, decData)` decompresses into a buffer the caller allocated. +- A command-line tool, `sz`: `-i` input, `-z` compressed file, `-o` decompressed file, `-f`/`-d` data type, `-1`â€Ļ`-4` dimensions, `-M `, and `-a` for error statistics. Most SZ2 command-line arguments are still accepted. `-h2` lists them. +- INI configuration files, passed with `-c`. The example is `test/sz.config`, now `tools/sz3/sz3.config`. +- OpenMP compression and decompression for all algorithms. Set `conf.openmp = true`, or `OpenMP = YES` in a configuration file. The data is split along its slowest dimension, and only the ABS and REL modes are supported with OpenMP. +- Error-bound modes ABS, REL, PSNR, NORM (L2 norm), ABS_AND_REL and ABS_OR_REL. On the command line the bounds can also be given with `-A`, `-R`, `-S` and `-N`. +- Integer data. The command line takes 32-bit and 64-bit integers with `-I 32` or `-I 64`. +- Text output of decompressed data, with `-t`. +- Data with more than four dimensions (folded into four). +- `make install` installs a CMake package. `find_package(SZ3)` provides the `SZ3::SZ3` target. +- CMake options `SZ3_USE_BUNDLED_ZSTD` and `SZ3_DEBUG_TIMINGS` (debug timings on by default). +- A polynomial regression predictor and a run-length encoder. +- The README lists the SZ2 command-line options SZ3 does not support (`-c` in SZ2 format, `-p`, `-T`, `-P`). + +**Changes** +- Headers moved from `include/` to `include/SZ3/`. Include them as `SZ3/...`. +- The compression configuration is stored at the end of the compressed data, so decompression needs only the compressed bytes. +- `sz` replaces `sz_demo`. `sz_pw` and `sz_truncate` became beta examples. +- The version macros are in the generated header `SZ3/version.hpp`. +- 3.1.1 changed the stored configuration (narrower field types and an `openmp` flag). Data compressed by 3.1.0 cannot be decompressed by 3.1.1 or later. +- 3.1.2 replaced the `#define` constants of 3.1.0 and 3.1.1 with enums in `namespace SZ`. `METHOD_*` became `ALGO_*` (the field `cmprMethod` became `cmprAlgo`), and `ABS`/`REL` became `EB_ABS`/`EB_REL`. +- 3.1.2 also renamed several `Config` fields, for example `enable_lorenzo` to `lorenzo`, `quant_state_num` to `quantbinCnt`, `block_size` to `blockSize`, and `interp_op` to `interpAlgo`. + +## 3.1.3.1 (2022-03-16) + +Tag `v3.1.3.1`, no GitHub release. + +**Fixes** +- `SZ_compress` no longer modifies the input data. +- Fixes to interpolation tuning and data sampling. +- The build works on 32-bit Windows, and `long` was replaced by `int64_t` for portability. + +## 3.1.4 (2022-04-09) + +No tag and no GitHub release. This is commit `75b2c7e`. + +**New** +- Builds and runs on Windows with MSYS2/MinGW (instructions: https://github.com/szcompressor/SZ3/issues/5#issuecomment-1094039224). Native Visual Studio builds came in 3.1.8. + +**Fixes** +- Fixes for OpenMP, the command line and the CMake build. + +## 3.1.5.1 (2022-04-29) + +Tag `v3.1.5.1`, no GitHub release. This entry also covers 3.1.5 (commit `be95b02`, 2022-04-28, no tag and no GitHub release), which was current for one day. + +**New** +- H5Z-SZ3, an HDF5 filter (ID 32024), built with `-DBUILD_H5Z_FILTER=ON`. `sz3ToHDF5` and `dsz3FromHDF5` are test tools. [#11](https://github.com/szcompressor/SZ3/pull/11) + +**Changes** +- The bundled Zstd moved to `tools/zstd` and was updated to 1.4.5. +- The examples moved from `test/` to `examples/`, and `sz.config` was renamed `sz3.config`. +- The generated `SZ3/version.hpp` is written to the build directory and installed. + +**Fixes** +- libpressio integration fixes. [#6](https://github.com/szcompressor/SZ3/pull/6) +- Explicit type in `std::multiplies`, for compilers that required it. [#10](https://github.com/szcompressor/SZ3/pull/10) +- A memory-allocation fix. + +## 3.1.7 (2022-11-12) + +Tag `v3.1.7`, no GitHub release. This entry also covers 3.1.5.4 (tag `v3.1.5.4`, 2022-10-24) and 3.1.6 (tag `v3.1.6`, 2022-11-01). Each was current for less than two weeks, and neither has a GitHub release. Versions 3.1.5.2 and 3.1.5.3 exist only as commits. + +**New** +- A C API in `tools/sz3c` (`SZ_compress_args`, `SZ_decompress`) with SZ2's signatures and constants. +- A Python API, `tools/pysz/pysz.py`, that loads the SZ3 or SZ2 shared library through ctypes. +- MDZ, a compressor for molecular-dynamics trajectories, in `tools/mdz`, built with `-DBUILD_MDZ=ON`. +- `print_h5repack_args` builds the HDF5 filter's `cd_values` for `h5repack`. +- A smoke test for `sz3`. + +**Changes** +- The command-line tool was renamed from `sz` to `sz3`. +- The tools and examples moved from `examples/` to `tools/sz3/`. +- CMake 3.18 or newer is required. +- The HDF5 filter's `cd_values` now use the same layout as H5Z-SZ, the SZ2 filter. +- Data with more than four dimensions is rejected instead of being folded into four. +- When GSL is found, SZ3 links it, and the installed CMake package looks for GSL. [#27](https://github.com/szcompressor/SZ3/pull/27) + +**Fixes** +- Fixes to OpenMP (also faster), `SZFastFrontend::size_est()`, the polynomial regression predictor and quantization-bin handling. +- Stride types are `size_t`. [#22](https://github.com/szcompressor/SZ3/pull/22) +- The H5Z-SZ3 header compiles as C, and the filter builds without warnings. [#21](https://github.com/szcompressor/SZ3/pull/21), [#25](https://github.com/szcompressor/SZ3/pull/25) + +## 3.1.8 (2023-11-30) + +Tag `v3.1.8`. The GitHub release has no notes and a source zip. + +**New** +- Native Windows builds with Visual Studio. With MSVC the bundled Zstd is used by default. [#29](https://github.com/szcompressor/SZ3/pull/29) +- The HDF5 filter reads a configuration file named by the `SZ3_CONFIG_PATH` environment variable. `cd_values` still take precedence. [#45](https://github.com/szcompressor/SZ3/pull/45) +- A faster mode for 1D Lorenzo. +- An absolute error bound of 0 compresses losslessly with Zstd. +- MDZ accepts 3D input. [#41](https://github.com/szcompressor/SZ3/pull/41) +- The Python API finds the library in a default location when no path is given. [#32](https://github.com/szcompressor/SZ3/pull/32) + +**Changes** +- The C++ namespace changed from `SZ` to `SZ3`, to avoid clashing with SZ2. [#44](https://github.com/szcompressor/SZ3/pull/44) +- `SZ3_DEBUG_TIMINGS` is off by default. +- OpenMP is optional: SZ3 links it only when it is found. +- The Python `decompress` returns data of the original dtype. [#43](https://github.com/szcompressor/SZ3/pull/43) + +**Fixes** +- `Config` fields have default values. [#47](https://github.com/szcompressor/SZ3/pull/47) +- `SZFastFrontend::size_est()` no longer returns an undefined value. [#46](https://github.com/szcompressor/SZ3/pull/46) +- Builds with GCC 13 (missing `uint8_t`). [#38](https://github.com/szcompressor/SZ3/pull/38) +- The HDF5 filter links MPI when HDF5 is parallel. [#37](https://github.com/szcompressor/SZ3/pull/37) +- Portable 64-bit integer types, and an uninitialized `max_err`. [#30](https://github.com/szcompressor/SZ3/pull/30), [#28](https://github.com/szcompressor/SZ3/pull/28) +- Fixes for OpenMP and interpolation+Lorenzo sampling. The "OpenMP threads" message is gone. + +## 3.2.0 (2024-08-16) + +Tag `v3.2.0`, no GitHub release. The v3.2.1 release notes list this version's changes. [#60](https://github.com/szcompressor/SZ3/pull/60) + +**New** +- The compressed data starts with a magic number and a data-format version. Decompression rejects data that SZ3 did not compress, or that has another data-format version, and names the version that wrote it. +- `SZ_compress(conf, data, cmpData, cmpCap)` compresses into a buffer the caller allocated and returns the compressed size. +- `ALGO_NOPRED`, which quantizes without prediction. +- Builds on 32-bit systems, such as wasm32, by using `std::unordered_map` there. [#56](https://github.com/szcompressor/SZ3/pull/56) +- Demos of custom pipelines in `tools/sz3/demo`. +- The README links a third-party Fortran API. + +**Changes** +- The internal API was restructured into decomposition, quantizer, encoder and lossless modules (`SZGenericCompressor`, `InterpolationDecomposition`, â€Ļ). Code written against the old frontend/predictor classes must be updated. +- `SZ_compress` and `SZ_decompress` throw `std::invalid_argument` when the output buffer is too small or the input is not SZ3 data. +- The HDF5 filter was rewritten. `cd_values` now hold the serialized SZ3 configuration, and the SZ2-style `cd_values` of 3.1.x are no longer understood. +- Data format 3.2.0. Data from 3.1.x cannot be decompressed. + +**Fixes** +- An integer overflow in quantization. +- Fixes for OpenMP, interpolation and MDZ (1D input, and the first frame counted in the compression ratio). + +## 3.2.1 (2024-10-03) + +Tag `v3.2.1`. The GitHub release lists #56 and #60, which were first in 3.2.0. + +**New** +- `ALGO_LOSSLESS`. SZ3 switches to Zstd only when the error bound is 0, when the output buffer is too small for lossy compression, or when Zstd gives the smaller output at a low compression ratio. +- The HDF5 filter has a README. `print_h5repack_args -c sz3.config` and `h5repack.sh` generate `h5repack` parameters from a configuration file. + +**Changes** +- The stored configuration is smaller, and the HDF5 filter's `cd_values` are compact. Data format 3.2.1: data from 3.2.0 cannot be decompressed. +- `IntegerQuantizer` was renamed `LinearQuantizer`. +- The code is formatted with clang-format and builds with stricter warnings. + +**Fixes** +- When the output buffer was too small, Zstd wrote a truncated stream without reporting an error. `Lossless_zstd` now checks the buffer size first. + +## 3.3.0 (2025-08-06) + +Tag `v3.3.0`. The GitHub release links [#94](https://github.com/szcompressor/SZ3/pull/94). + +**New** +- The interpolation algorithms gain key QoZ 1 and 2 features, anchor points and level-wise error bounds, which improve speed and quality. New configuration keys: `InterpolationAnchorStride`, `InterpolationAlpha` and `InterpolationBeta`. The complete QoZ is on the `QoZ` branch. [#94](https://github.com/szcompressor/SZ3/pull/94) +- Lorenzo and regression run on a new blockwise iterator. +- `sz3 -p` prints the configuration stored in a compressed file, and `sz3 -v` also prints the data-format version. +- `SZ_compress_size_bound()` gives the output-buffer size `SZ_compress` needs. +- CMake options `BUILD_TESTING` (unit and integration tests under `tools/test`) and `SZ3_USE_SKA_HASH`. +- The C API exports its functions from a Windows DLL and adds `free_buf`. +- The README links the third-party Rust and numcodecs APIs. + +**Changes** +- The INI parser is SZ3's own. Section names, keys and enum values are case-insensitive, and inih is no longer bundled. +- The polynomial regression, meta-Lorenzo and meta-regression predictors were removed, along with `SZIterateCompressor` and `LorenzoRegressionDecomposition` (replaced by `BlockwiseDecomposition`). The demos became `tools/sz3/sz3_customized_demo.cpp`. +- Dimensions of size 1 are dropped from `Config`. +- `ENABLE_GSL` was renamed `SZ3_ENABLE_GSL`. [#100](https://github.com/szcompressor/SZ3/pull/100) +- The API throws exceptions and writes to stderr, instead of calling `exit(0)`. +- Data format 3.3.0: data from 3.2.x cannot be decompressed. + +**Fixes** +- NaN values are stored exactly instead of being replaced by predictions. [#91](https://github.com/szcompressor/SZ3/pull/91) +- Compressing small arrays no longer writes past the buffer. [#70](https://github.com/szcompressor/SZ3/pull/70) +- `SZ_decompress` takes the compressed data as `const`. [#73](https://github.com/szcompressor/SZ3/pull/73) +- MSVC builds again. [#74](https://github.com/szcompressor/SZ3/pull/74) +- A small AddressSanitizer fix. [#96](https://github.com/szcompressor/SZ3/pull/96) +- The Python API copies buffers faster. [#99](https://github.com/szcompressor/SZ3/pull/99) +- Fixes for the regression predictor, OpenMP and the HDF5 filter. + +From December 2024 until 3.3.0, master reported version 3.2.2. That version was never tagged or released. + +## 3.3.1 (2025-10-23) + +No tag and no GitHub release. This is commit `f7f6989` ([#106](https://github.com/szcompressor/SZ3/pull/106)). pysz 1.0.2 was built from it. + +**New** +- pysz 1.0, a new Python package (Cython) published on PyPI: `pip install pysz`. [#106](https://github.com/szcompressor/SZ3/pull/106) +- CMake option `BUILD_SZ3_BINARY` (default ON) for the `sz3` tool and the C API. + +**Fixes** +- A sampling bug in interpolation tuning. [#104](https://github.com/szcompressor/SZ3/pull/104) +- OpenMP decompression failed when a thread's slice had fewer dimensions than the whole array. [#105](https://github.com/szcompressor/SZ3/pull/105) + +The data format is still 3.3.0, so 3.3.1 and 3.3.0 read each other's data. + +## 3.3.2 (2025-11-19) + +Tag `v3.3.2`. The GitHub release lists #104–#116, including the 3.3.1 changes. + +**New** +- `ALGO_BIOMD` and `ALGO_BIOMDXTC`, for molecular-dynamics data, plus a Huffman encoder with less storage overhead. [#115](https://github.com/szcompressor/SZ3/pull/115) +- SZ3Reader, a ParaView plugin that opens SZ3-compressed files, built with `-DBUILD_PARAVIEW_PLUGIN=ON`. [#112](https://github.com/szcompressor/SZ3/pull/112) +- The HDF5 filter builds on Windows with Visual Studio. CI builds and tests SZ3 with both Visual Studio and MinGW. [#107](https://github.com/szcompressor/SZ3/pull/107) +- `cdvalueHelper` converts between an SZ3 configuration and the filter's `cd_values`. [#109](https://github.com/szcompressor/SZ3/pull/109) +- pysz 1.0.3. + +**Changes** +- The compressed layout is now a 16-byte header (magic number, data-format version, payload size), then the payload, then the configuration. Data format 3.3.2: data from 3.3.0 and 3.3.1 cannot be decompressed. [#116](https://github.com/szcompressor/SZ3/pull/116) +- Compressed data is little-endian on every host, including big-endian ones. [#109](https://github.com/szcompressor/SZ3/pull/109) + +**Fixes** +- A bug in the compressed format; the PR does not describe it further. [#116](https://github.com/szcompressor/SZ3/pull/116) +- The HDF5 filter works on Windows. [#107](https://github.com/szcompressor/SZ3/pull/107) + +## 3.4.0 (2026-10-05) + +Tag `v3.4.0`. The GitHub release lists #127–#183. + +**New** +- One Huffman encoder for every algorithm, replacing the two earlier ones. Its code table takes about 0.3–0.7 bytes per symbol, where the one most algorithms used took 9–26. On 81 real fields from 32 datasets, the compression ratio is 1.5% higher and decompression 1.8 times faster (geometric means). [#163](https://github.com/szcompressor/SZ3/pull/163) +- Every build decompresses to the same values, whether or not the compiler fuses multiply-adds (Apple Silicon, aarch64, `-march=native`, Spack's default) or uses fast math (Clang, icx and MSVC fast modes; GCC's `-ffast-math` and x87 are refused at compile time). [#162](https://github.com/szcompressor/SZ3/pull/162), [#166](https://github.com/szcompressor/SZ3/pull/166) +- An installed SZ3 works through `find_package(SZ3)`. It provides `SZ3::SZ3core` (SZ3 and Zstd) and `SZ3::SZ3` (plus OpenMP when SZ3 was built with it), and `find_package(SZ3 COMPONENTS hdf5sz3)` requires the HDF5 filter's `SZ3::hdf5sz3`. Added with `add_subdirectory` or FetchContent, SZ3 leaves the parent's build settings alone and installs nothing. [#125](https://github.com/szcompressor/SZ3/pull/125), [#126](https://github.com/szcompressor/SZ3/pull/126), [#127](https://github.com/szcompressor/SZ3/pull/127), [#128](https://github.com/szcompressor/SZ3/pull/128), [#150](https://github.com/szcompressor/SZ3/pull/150), [#160](https://github.com/szcompressor/SZ3/pull/160), [#161](https://github.com/szcompressor/SZ3/pull/161) +- HDF5 filter: `H5Pset_sz3()` sets the filter from C. The filter's `cd_values` carry the data version, each chunk is decompressed with the configuration stored in it, and the filter does not use OpenMP. See [tools/H5Z-SZ3/README.md](tools/H5Z-SZ3/README.md). [#155](https://github.com/szcompressor/SZ3/pull/155), [#161](https://github.com/szcompressor/SZ3/pull/161) +- A build without OpenMP decompresses data compressed with OpenMP. [#160](https://github.com/szcompressor/SZ3/pull/160) +- OpenMP splits `ALGO_BIOMD` data `{frames, atoms, 3}` along the atoms, so every part keeps all frames: with 16 threads and 20 frames per chunk, systems of 60,000 to 200,000 atoms lose at most 0.5% of the compression ratio, where splitting along the frames lost 3% to 14%. [#182](https://github.com/szcompressor/SZ3/pull/182) +- A Nix expression for NixOS. [#129](https://github.com/szcompressor/SZ3/pull/129) +- `ALGO_BIOMD` is a new codec for molecular-dynamics coordinates `{frames, atoms, 3}` with an absolute bound: rigid water and bonded atoms are predicted from their geometry on an integer lattice. On 16 GROMACS trajectories at 5e-4 nm its compression ratio is 1.23 times that of the previous `ALGO_BIOMD` with one frame per chunk and 1.29 times with 20. Other shapes throw `std::invalid_argument` (in HDF5, creating the dataset fails); coordinates beyond its lattice (2048 nm at 5e-4 nm and 128 nm at 5e-5 nm for float) and NaN or Inf outside trailing fill frames go to lossless compression. [#169](https://github.com/szcompressor/SZ3/pull/169) +- [docs/molecular-dynamics.md](docs/molecular-dynamics.md) for molecular-dynamics data, and the [advanced guide](docs/advanced-guide.md) for algorithms, building, CMake and packaging, the C++ API, OpenMP, floating point and contributing; the HDF5 filter's README covers using it in an application; the README is a quick start. [#173](https://github.com/szcompressor/SZ3/pull/173) +- `find_package(SZ3 COMPONENTS SZ3c)` provides `SZ3::SZ3c`, the SZ2-compatible C library. [#177](https://github.com/szcompressor/SZ3/pull/177) +- Homebrew: `brew install szcompressor/tap/sz3`. [#176](https://github.com/szcompressor/SZ3/pull/176) +- pysz builds against an installed SZ3 and its Zstd with `PYSZ_SZ3_PREFIX`. [#177](https://github.com/szcompressor/SZ3/pull/177) + +**Compatibility** +- Data format 3.4.0. Data from 3.3.2 decompresses, except data compressed with `ALGO_NOPRED` or the molecular-dynamics algorithms, which is refused. Data from 3.3.1 and earlier cannot be decompressed. [#163](https://github.com/szcompressor/SZ3/pull/163) +- `SZBioMDDecomposition` and `make_decomposition_biomd` take no quantizer. [#169](https://github.com/szcompressor/SZ3/pull/169) +- Data compressed by an earlier version built with FMA may exceed the error bound when decompressed by 3.4.0; decompress it with the build that compressed it. [#162](https://github.com/szcompressor/SZ3/pull/162) +- CMake 3.19 or newer is required. `find_package(SZ3 )` accepts only SZ3 versions with the same major version. [#161](https://github.com/szcompressor/SZ3/pull/161) +- Removed: `ArithmeticEncoder`, `RunlengthEncoder`, `SZTruncateCompressor` and the preprocessors, which no build target used, the bundled ska hash map, and `HuffmanEncoderV2`. [#150](https://github.com/szcompressor/SZ3/pull/150), [#157](https://github.com/szcompressor/SZ3/pull/157), [#163](https://github.com/szcompressor/SZ3/pull/163) +- Removed: `ALGO_BIOMDXTC` and its LGPL encoder, so SZ3 is BSD only; use `ALGO_BIOMD`. The value stays defined, and compressing with it throws `std::invalid_argument` (`H5Pset_sz3` returns an error). Data compressed with it by 3.3.2 no longer decompresses. `LinearQuantizer` no longer takes a third argument. [#183](https://github.com/szcompressor/SZ3/pull/183) +- `Config::load` and the lossless stages take the length of the buffer they read. [#145](https://github.com/szcompressor/SZ3/pull/145) +- Integer data compressed by earlier versions decompresses only with those versions. 8-byte integers beyond Âą2^53 are refused. [#165](https://github.com/szcompressor/SZ3/pull/165) +- `cmake --install` no longer installs `sz3_customized_demo` or `share/SZ3/testfloat_8_8_128.dat`. +- `libhdf5sz3` and `libSZ3c` export only their public functions, and carry an SOVERSION. [#158](https://github.com/szcompressor/SZ3/pull/158), [#159](https://github.com/szcompressor/SZ3/pull/159) +- A static `libhdf5sz3` keeps `H5Pset_sz3`, `set_SZ3_conf_to_H5` and `get_SZ3_conf_from_H5` hidden in the shared library it is linked into, so they cannot take over another SZ3 copy's calls. `H5PLget_plugin_type` and `H5PLget_plugin_info` stay exported, as HDF5's `H5PLextern.h` declares them. [#174](https://github.com/szcompressor/SZ3/pull/174) +- 32-bit builds (wasm32) and 64-bit builds decompress each other's data: every size in the stream is 8 bytes, as 64-bit builds already wrote it. [#177](https://github.com/szcompressor/SZ3/pull/177) +- `Config::load` skips fields that a later version appends to the end of a config. [#178](https://github.com/szcompressor/SZ3/pull/178) + +**Fixes** +- Damaged or truncated compressed data makes decompression throw instead of reading past the buffer, and the HDF5 filter reports the error instead of ending the program. [#121](https://github.com/szcompressor/SZ3/pull/121), [#145](https://github.com/szcompressor/SZ3/pull/145), [#149](https://github.com/szcompressor/SZ3/pull/149) +- OpenMP: compression no longer writes an undecodable stream or hangs when a region gets fewer threads than asked or a chunk fails. [#149](https://github.com/szcompressor/SZ3/pull/149), [#161](https://github.com/szcompressor/SZ3/pull/161) +- `ALGO_LORENZO_REG` could write past its buffer with a small `BlockSize`, and with only regression enabled it read past its buffer on 3D and 4D data. [#151](https://github.com/szcompressor/SZ3/pull/151), [#161](https://github.com/szcompressor/SZ3/pull/161) +- Integer data is compressed as floating point and rounded back, so values near the type's limits no longer come back past the bound, and REL and PSNR bounds work on data whose range the type cannot hold. [#165](https://github.com/szcompressor/SZ3/pull/165) +- Float data could come back a fraction of an ulp past the bound: `LinearQuantizer` checked the error after rounding it to float. [#166](https://github.com/szcompressor/SZ3/pull/166) +- The HDF5 filter refuses datatypes it cannot compress, instead of writing data that comes back wrong. [#158](https://github.com/szcompressor/SZ3/pull/158) +- The HDF5 filter keeps `H5Z_FLAG_OPTIONAL` (as h5py sets it), and refuses a chunk whose size a filter before it changed. [#163](https://github.com/szcompressor/SZ3/pull/163) +- SZ3 builds for wasm32 again. [#117](https://github.com/szcompressor/SZ3/pull/117) +- An error bound that is negative or not finite (also a relative bound over data with Inf) compresses losslessly with every algorithm, as a bound of 0 did. [#169](https://github.com/szcompressor/SZ3/pull/169) +- `ALGO_BIOMD` stores a chunk with NaN or Inf losslessly also when built with GCC's `-funsafe-math-optimizations` and `-ffinite-math-only` for an FMA target, which made it code NaN as 0. [#171](https://github.com/szcompressor/SZ3/pull/171) +- `cdvalueHelper.py` writes `cd_values` with the data version first, as the filter reads them; `mdz` accepts 3D input in Debug builds and refuses 1D input, which it could not compress. [#173](https://github.com/szcompressor/SZ3/pull/173) +- `libSZ3c` returns NULL on an error instead of ending the program with `exit(0)`, and `SZ_decompress` checks the dimensions against the data. [#177](https://github.com/szcompressor/SZ3/pull/177) +- `Lossless_zstd::compress` refuses a buffer under 8 bytes instead of writing past it. [#177](https://github.com/szcompressor/SZ3/pull/177) +- `ALGO_BIOMD` refuses a corrupt mode byte or box flag instead of decoding with it. [#178](https://github.com/szcompressor/SZ3/pull/178) +- Warnings under GCC 16. [#178](https://github.com/szcompressor/SZ3/pull/178) + +**Compared with other compressors** + +Single-threaded on an Intel i9-13900K (GCC 13, `-O3`). SZ3 uses its default `ALGO_INTERP_LORENZO`; every compressor runs in its pointwise error mode with the bound REL × (max − min). Geometric means over five float fields: Hurricane P, Miranda density, NYX temperature, SCALE QV and CESM U. Speeds are in original bytes per second, with 1 MB = 10^6 bytes. Versions: ZFP develop `c0c2c40`, SPERR 0.8.4 `4061a44`, MGARD 1.7.0 `6b43196` (MGARD is `mgard::compress`, MGARD-X its serial device), SZx `3a0f875`, SZp `5d15dc3`. ZFP and MGARD stay at 2–70% of the bound, which lowers their ratios. + +| Compression ratio | REL 1e-2 | REL 1e-3 | REL 1e-4 | +|---|---|---|---| +| SZ3 3.4.0 | 1295.5 | 150.5 | 32.2 | +| SPERR | 1026.6 | 179.3 | 46.2 | +| ZFP | 26.6 | 13.0 | 6.9 | +| MGARD | 12.2 | 6.0 | 4.0 | +| MGARD-X | 20.3 | 12.0 | 6.6 | +| SZx | 21.4 | 6.5 | 4.0 | +| SZp | 36.4 | 14.0 | 8.5 | + +| Compression speed (MB/s) | REL 1e-2 | REL 1e-3 | REL 1e-4 | +|---|---|---|---| +| SZ3 3.4.0 | 507 | 474 | 434 | +| SPERR | 153 | 137 | 108 | +| ZFP | 493 | 377 | 300 | +| MGARD | 56 | 57 | 54 | +| MGARD-X | 93 | 93 | 92 | +| SZx | 1706 | 1179 | 1124 | +| SZp | 3246 | 1630 | 1261 | + +| Decompression speed (MB/s) | REL 1e-2 | REL 1e-3 | REL 1e-4 | +|---|---|---|---| +| SZ3 3.4.0 | 1495 | 1197 | 980 | +| SPERR | 299 | 268 | 214 | +| ZFP | 1667 | 1109 | 749 | +| MGARD | 49 | 44 | 38 | +| MGARD-X | 92 | 89 | 84 | +| SZx | 2873 | 1716 | 1584 | +| SZp | 7161 | 3583 | 2601 | diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..ea4136e6 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,145 @@ +cmake_minimum_required(VERSION 3.19) +project(SZ3 VERSION 3.4.0) + +#data version defines the version of the compressed data format +#it is not always equal to the program version (e.g., SZ3 v3.1.0 and SZ3 v.3.1.1 may use the same data version of v.3.1.0) +#only update data version if the new version of the program changes compressed data format +set(SZ3_DATA_VERSION 3.4.0) + +include(GNUInstallDirs) +# Off when SZ3 is added with add_subdirectory or FetchContent: the parent installs what it needs of SZ3. +string(COMPARE EQUAL "${CMAKE_SOURCE_DIR}" "${PROJECT_SOURCE_DIR}" SZ3_IS_TOP_LEVEL) +option(SZ3_INSTALL "install SZ3's headers, libraries, tools and CMake package" ${SZ3_IS_TOP_LEVEL}) + + +# Cache defaults, so only at top level: set from a subproject they would apply to the parent's whole build. +if (SZ3_IS_TOP_LEVEL) + option(BUILD_SHARED_LIBS "build shared libraries by default" ON) + set(default_build_type "Release") + if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "No build type was set. Setting build type to ${default_build_type}.") + set(CMAKE_BUILD_TYPE ${default_build_type} CACHE + STRING "Choose the type to build" FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "RelWithDebInfo") + endif () + option(BUILD_TESTING "Build tests" OFF) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +endif () +if (MSVC) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /bigobj") +else () #-Wunused-parameter + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wreorder -Wsign-compare -Wsuggest-override -Wold-style-cast -Wcast-qual -Wextra-semi -Wnon-virtual-dtor -Wcast-align -Wunused-variable") +endif () + +find_package(PkgConfig) + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in + ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp) + +# SZ3::SZ3core is SZ3 with only the dependencies it cannot work without. SZ3::SZ3 adds the optional +# ones SZ3 was built with, such as OpenMP. SZ3's headers still use OpenMP in a consumer compiled with it. +add_library(SZ3core INTERFACE) +add_library(SZ3::SZ3core ALIAS SZ3core) +target_include_directories( + SZ3core INTERFACE + "$" + "$" + "$" +) +target_compile_features(SZ3core + INTERFACE cxx_std_17 +) + +add_library(${PROJECT_NAME} INTERFACE) +add_library(SZ3::SZ3 ALIAS ${PROJECT_NAME}) +target_link_libraries(${PROJECT_NAME} INTERFACE SZ3core) +find_package(OpenMP) +if (OpenMP_FOUND) + # BUILD_INTERFACE only: SZ3Config.cmake names the OpenMP target for the language the consumer + # actually enabled. Exported as it stands, OpenMP::OpenMP_CXX is a target a C-only consumer + # cannot be given, and CMake ends the consumer's configure over it. + target_link_libraries(${PROJECT_NAME} INTERFACE $) +endif () + +option(SZ3_DEBUG_TIMINGS "print debug timing information" OFF) +if (SZ3_DEBUG_TIMINGS) + target_compile_definitions(SZ3core INTERFACE SZ3_DEBUG_TIMINGS=1) +else () + target_compile_definitions(SZ3core INTERFACE SZ3_DEBUG_TIMINGS=0) +endif () + +# "${MSVC}" reads as ON under MSVC and OFF everywhere else. Keep the quotes. +option(SZ3_USE_BUNDLED_ZSTD "prefer the bundled version of Zstd" "${MSVC}") +pkg_search_module(ZSTD libzstd) +if (ZSTD_FOUND AND NOT SZ3_USE_BUNDLED_ZSTD) + # Not PkgConfig::ZSTD: that imported target does not exist in the consumer's project. + # BUILD_INTERFACE, and no include directory: both are paths on this machine only. + target_link_libraries(SZ3core INTERFACE $) +else () + # What this build uses, for SZ3Config.cmake: the option only states a preference. + set(SZ3_USE_BUNDLED_ZSTD ON) + add_subdirectory(tools/zstd) + target_link_libraries(SZ3core INTERFACE sz3_zstd) +endif () + +# Only at top level: a parent's BUILD_TESTING is for its own tests, and SZ3's would fetch a second googletest. +if (BUILD_TESTING AND SZ3_IS_TOP_LEVEL) + enable_testing() + add_subdirectory(tools/test) +endif () + + +option(BUILD_SZ3_BINARY "build SZ3 tools" ${SZ3_IS_TOP_LEVEL}) +if (BUILD_SZ3_BINARY) + add_subdirectory(tools/sz3) + add_subdirectory(tools/sz3c) +endif () + +option(BUILD_H5Z_FILTER "build the H5Z-SZ3 filter" OFF) +if (BUILD_H5Z_FILTER) + add_subdirectory(tools/H5Z-SZ3) +endif () + +option(BUILD_MDZ "build MDZ (MMD-SZ)" OFF) +if (BUILD_MDZ) + add_subdirectory(tools/mdz) +endif () + +option(BUILD_PARAVIEW_PLUGIN "build the ParaView plugin" OFF) +if (BUILD_PARAVIEW_PLUGIN) + add_subdirectory(tools/paraview) +endif () + +if (SZ3_INSTALL) + install(TARGETS ${PROJECT_NAME} SZ3core + EXPORT SZ3Targets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + PATTERN "version.hpp.in" EXCLUDE) + install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/SZ3/" + ) + install(EXPORT SZ3Targets NAMESPACE SZ3:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3) + include(CMakePackageConfigHelpers) + configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/SZ3Config.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3 + ) + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion + ) + install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/SZ3Config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/SZ3ConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3 + ) +endif () +#export sz3 target for external use +export(TARGETS SZ3 SZ3core FILE SZ3.cmake) diff --git a/README.md b/README.md new file mode 100644 index 00000000..54a601ab --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +SZ3: A Modular Error-bounded Lossy Compression Framework for Scientific Datasets +===== + +SZ3 compresses floating-point and integer arrays from simulations and instruments, and guarantees that every +decompressed value differs from the original by no more than the error bound you set. It is a header-only C++17 +library. It can also be used from C, Python, the `sz3` command line, and HDF5 through a filter. + +## Installation + +With Homebrew (macOS and Linux): `brew install szcompressor/tap/sz3`. + +To build from source you need a C++17 compiler and CMake 3.19 or newer. Zstd and OpenMP are used when found. + +```bash +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX= +cmake --build build -j +cmake --install build +``` +The tools go to `/bin` and the headers to `/include`. + +## Quick start + +### Command line + +```bash +# Compress 8x8x128 float data (dimensions fastest-varying first) with a relative error bound of 1e-3 +sz3 -f -i tools/sz3/testfloat_8_8_128.dat -z test.sz -3 8 8 128 -M REL 1e-3 +# Decompress to test.out; -a compares with the original (-i) and prints the maximum error and compression ratio +sz3 -f -z test.sz -o test.out -i tools/sz3/testfloat_8_8_128.dat -a +``` +Run `sz3 -h` for every option; the common ones: + +| Option | Meaning | +|---|---| +| `-f`, `-d`, `-I 32`, `-I 64` | data type: float, double, int32, int64 | +| `-i ` | original data, raw binary | +| `-z ` | compressed file: written when compressing, read when decompressing | +| `-o ` | decompressed data, raw binary (`-t` writes text) | +| `-1 nx`, `-2 nx ny`, `-3 nx ny nz`, `-4 nx ny nz np` | dimensions, fastest-varying first: `-3 nx ny nz` is `data[nz][ny][nx]` | +| `-M ` | error-bound mode and bound: `ABS`, `REL`, `PSNR`, `NORM`; `ABS_AND_REL` and `ABS_OR_REL` take `-A -R ` | +| `-c ` | configuration file, such as [tools/sz3/sz3.config](tools/sz3/sz3.config); choose the algorithm here | +| `-a` | after decompression, print error statistics (needs `-i`) | +| `-p` | print the configuration of the compressed data | +| `-v` | print the SZ3 version and the data-format version | + +### C++ + +```cpp +#include +#include + +int main() { + std::vector data(100 * 200 * 300, 1.0f); + SZ3::Config conf(100, 200, 300); // 300 is the fastest-varying dimension + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 1e-3; + + size_t cmpSize; + char *cmpData = SZ_compress(conf, data.data(), cmpSize); + + SZ3::Config decConf; // filled from the compressed data + float *decData = SZ_decompress(decConf, cmpData, cmpSize); + + delete[] cmpData; + delete[] decData; +} +``` + +In a CMake project, add `` to `CMAKE_PREFIX_PATH`, then `find_package(SZ3)` and link `SZ3::SZ3`. + +## Interfaces + +| Interface | How to use / where | Maintained by | +|---|---|---| +| C++ | `#include `; see [above](#c) | SZ3 | +| C | [tools/sz3c/include/sz3c.h](tools/sz3c/include/sz3c.h), library `SZ3c`; SZ2-compatible functions | SZ3 | +| Python | `pip install pysz`; [tools/pysz](tools/pysz/README.md) | SZ3 | +| Command line | `sz3`; see [above](#command-line) | SZ3 | +| HDF5 filter | H5Z-SZ3, filter ID 32024; [tools/H5Z-SZ3](tools/H5Z-SZ3/README.md) | SZ3 | +| ParaView | SZ3Reader plugin; [tools/paraview](tools/paraview/README.md) | SZ3 | +| Fortran | [ofmla/sz3_simple_example](https://github.com/ofmla/sz3_simple_example) | [Oscar Mojica](https://github.com/ofmla) | +| Rust | [sz3-rs](https://github.com/apertus-open-source-cinema/sz3-rs) | [Juniper Tyree](https://github.com/juntyr) and [Robin Heinemann](https://github.com/rroohhh) | +| Python numcodecs | [numcodecs-rs codecs/sz3](https://github.com/juntyr/numcodecs-rs/blob/main/codecs/sz3/) | [Juniper Tyree](https://github.com/juntyr) | + +## Further documentation + +* [docs/advanced-guide.md](docs/advanced-guide.md): algorithms and error-bound modes, build options, CMake and + packaging, the C++ API in depth, OpenMP, floating point, and the rules for contributors. +* [docs/molecular-dynamics.md](docs/molecular-dynamics.md): molecular-dynamics data and GROMACS. +* [CHANGELOG.md](CHANGELOG.md): version history and which data formats each version reads. + +## Citing SZ3 + +[//]: # (**Kindly note**: If you mention SZ3 in your paper, the most appropriate citation is to include these three references (**TBD22, ICDE21, Bigdata18**) because they cover the design and implementation of the latest version of SZ.) +* QOZv2 (the enhanced interpolation-based algorithm): [High-performance Effective Scientific Error-bounded Lossy Compression with Auto-tuned Multi-component Interpolation](https://dl.acm.org/doi/10.1145/3639259). +* SZ3's interpolation-based algorithm: [Optimizing Error-Bounded Lossy Compression for ScientiīŦc Data by Dynamic Spline Interpolation](https://ieeexplore.ieee.org/document/9458791). +* The software engineering design of SZ3: [SZ3: A modular framework for composing prediction-based error-bounded lossy compressors](https://ieeexplore.ieee.org/abstract/document/9866018). + +## License and contact + +SZ3 is released under a BSD license; see [copyright-and-BSD-license.txt](copyright-and-BSD-license.txt). +The vendored Zstd in `tools/zstd` keeps its own license. + +* Lead developer and maintainer: Kai Zhao +* Contributors: Robert Underwood, Xin Liang, Jinyang Liu, Sheng Di, and [everyone else on GitHub](https://github.com/szcompressor/SZ3/graphs/contributors) +* SZ project lead: Franck Cappello + +Report bugs and ask questions in [GitHub issues](https://github.com/szcompressor/SZ3/issues). diff --git a/SZ3Config.cmake.in b/SZ3Config.cmake.in new file mode 100644 index 00000000..731f5a8b --- /dev/null +++ b/SZ3Config.cmake.in @@ -0,0 +1,76 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) + +# Every search that can decline runs before any SZ3:: target exists, so a decline leaves none behind +# for a consumer's own copy of SZ3 to collide with. +if(NOT "@SZ3_USE_BUNDLED_ZSTD@") + # Found again, not taken from the export: the build machine's path need not exist here. + find_library(SZ3_ZSTD_LIBRARY NAMES zstd) + if(NOT SZ3_ZSTD_LIBRARY) + set(SZ3_NOT_FOUND_MESSAGE "SZ3 was built against the system Zstd and no libzstd was found here. Install Zstd's development package, or set SZ3_ZSTD_LIBRARY.") + set(SZ3_FOUND FALSE) + return() + endif() +endif() + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/HDF5SZ3.cmake") + # Not again when the consumer already found it: FindHDF5 would reset its HDF5_* variables to C only. + if(NOT TARGET HDF5::HDF5) + # find_dependency, so this does not demand HDF5 on its own: a consumer who did not write + # REQUIRED gets SZ3_FOUND FALSE and a reason. + find_dependency(HDF5 COMPONENTS C) + endif() + set(SZ3_hdf5sz3_FOUND TRUE) +endif() +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/SZ3c.cmake") + set(SZ3_SZ3c_FOUND TRUE) +endif() +if(@OpenMP_FOUND@) + # QUIET and never declining: SZ3::SZ3 goes without OpenMP when the consumer's compiler has none, and + # SZ3::SZ3core and SZ3::hdf5sz3 do not use it. Asked per language, because FindOpenMP fails outright + # when it searches for one the consumer did not enable. C and C++ share one runtime everywhere SZ3 + # builds, so either will do. + get_property(sz3_enabled_languages GLOBAL PROPERTY ENABLED_LANGUAGES) + if("CXX" IN_LIST sz3_enabled_languages) + find_package(OpenMP QUIET COMPONENTS CXX) + if(OpenMP_CXX_FOUND) + set(sz3_openmp OpenMP::OpenMP_CXX) + endif() + elseif("C" IN_LIST sz3_enabled_languages) + find_package(OpenMP QUIET COMPONENTS C) + if(OpenMP_C_FOUND) + set(sz3_openmp OpenMP::OpenMP_C) + endif() + endif() + unset(sz3_enabled_languages) +endif() +check_required_components(SZ3) +# find_package clears a false SZ3_FOUND before running this file, so only check_required_components +# can have set it false here. +if(DEFINED SZ3_FOUND AND NOT SZ3_FOUND) + return() +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/SZ3Targets.cmake") +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/HDF5SZ3.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/HDF5SZ3.cmake") +endif() +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/SZ3c.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/SZ3c.cmake") +endif() +if(NOT "@SZ3_USE_BUNDLED_ZSTD@") + set_property(TARGET SZ3::SZ3core APPEND PROPERTY INTERFACE_LINK_LIBRARIES "${SZ3_ZSTD_LIBRARY}") +endif() +if(sz3_openmp) + set_property(TARGET SZ3::SZ3 APPEND PROPERTY INTERFACE_LINK_LIBRARIES ${sz3_openmp}) + unset(sz3_openmp) +endif() + +# Without these, target_link_libraries(x PRIVATE hdf5sz3) silently becomes -lhdf5sz3. +foreach(sz3_target SZ3 hdf5sz3) + if(TARGET SZ3::${sz3_target} AND NOT TARGET ${sz3_target}) + add_library(${sz3_target} INTERFACE IMPORTED) + target_link_libraries(${sz3_target} INTERFACE SZ3::${sz3_target}) + endif() +endforeach() diff --git a/copyright-and-BSD-license.txt b/copyright-and-BSD-license.txt new file mode 100644 index 00000000..e48a51c5 --- /dev/null +++ b/copyright-and-BSD-license.txt @@ -0,0 +1,29 @@ +Copyright Š 2016 , UChicago Argonne, LLC +All Rights Reserved + +SZ licence is under the BSD license. Details are shown below. + +OPEN SOURCE LICENSE (license number: SF-16-105) + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Software changes, modifications, or derivative works, should be noted with comments and the author and organization's name. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the names of UChicago Argonne, LLC or the Department of Energy nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +4. The software and the end-user documentation included with the redistribution, if any, must include the following acknowledgment: + + "This product includes software produced by UChicago Argonne, LLC under Contract No. DE-AC02-06CH11357 with the Department of Energy." + +****************************************************************************************************** +DISCLAIMER + +THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. + +NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. + +*************************************************************************************************** + +Contact: Sheng Di (sdi1@anl.gov), Franck Cappello(cappello@mcs.anl.gov) diff --git a/default.nix b/default.nix new file mode 100644 index 00000000..d15723e1 --- /dev/null +++ b/default.nix @@ -0,0 +1,36 @@ +with import { }; +stdenv.mkDerivation { + pname = "sz3"; + version = "3.3.x"; + + src = fetchFromGitHub { + owner = "szcompressor"; + repo = "SZ3"; + rev = "34b36f2af0d43aa4f5dea0e39500e91f6d35c9cf"; + sha256 = "0fckf5939m9d2w3095isnq3qfrz8fxpppn7y3c2hwzdh72vdzic7"; + }; + + nativeBuildInputs = [ + cmake + pkg-config + ]; + + buildInputs = [ + zstd + ] + ++ lib.optionals stdenv.cc.isClang [ + llvmPackages.openmp + ]; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=ON" + ]; + + meta = with lib; { + description = "A modular error-bounded lossy compression framework for scientific datasets"; + homepage = "https://github.com/szcompressor/SZ3"; + maintainer = [ "szcomprssor" ]; + license = licenses.bsd3; + platforms = platforms.unix; + }; +} diff --git a/docs/advanced-guide.md b/docs/advanced-guide.md new file mode 100644 index 00000000..4459fbc1 --- /dev/null +++ b/docs/advanced-guide.md @@ -0,0 +1,301 @@ +# SZ3 advanced guide + +This guide is for choosing an algorithm and error bound, building and packaging SZ3, using its C++ API beyond the +defaults, and changing SZ3's code. To compress with the default settings, the [README](../README.md) is enough. + +## Algorithms and error-bound modes + +Set the algorithm with `Config::cmprAlgo`, or `CmprAlgo` in a configuration file. + +| Algorithm | Use it for | +|---|---| +| `ALGO_INTERP_LORENZO` (default) | Most data. It tunes interpolation and Lorenzo prediction on a sample of the data and keeps whichever is better. | +| `ALGO_INTERP` | Interpolation with the parameters you set, without auto-tuning; for users who tune those parameters themselves. | +| `ALGO_LORENZO_REG` | Blockwise Lorenzo and regression prediction, the SZ2 algorithm. | +| `ALGO_NOPRED` | Quantization without prediction: a fast baseline. | +| `ALGO_LOSSLESS` | Zstd only. SZ3 also switches to it by itself when the error bound is 0, negative or not finite, or when Zstd alone gives a smaller result. | +| `ALGO_BIOMD` | Molecular-dynamics coordinates, `{frames, atoms, 3}`. See [molecular-dynamics.md](molecular-dynamics.md). | + +Set the error-bound mode with `Config::errorBoundMode`, or `-M` on the command line. + +| Mode | Bound | +|---|---| +| `EB_ABS` (`ABS`) | Every value is within `absErrorBound` of the original. | +| `EB_REL` (`REL`) | Within `relErrorBound` × (max − min) of the data. | +| `EB_ABS_AND_REL` / `EB_ABS_OR_REL` | The smaller / the larger of the two bounds above. | +| `EB_PSNR` (`PSNR`), `EB_L2NORM` (`NORM`) | A target PSNR or L2-norm error. SZ3 converts it to an absolute bound, which is the pointwise guarantee. | + +## Building + +Requirements: a C++17 compiler and CMake 3.19 or newer. Zstd, OpenMP and HDF5 are optional (see +[Dependencies](#dependencies)). + +| Option | Default | Effect | +|---|---|---| +| `BUILD_SHARED_LIBS` | ON at top level; the parent's in a subproject | shared `libhdf5sz3` and `libSZ3c`; OFF builds static ones | +| `BUILD_SZ3_BINARY` | ON at top level, OFF in a subproject | `sz3`, `sz3_smoke_test`, the C API library `libSZ3c`, the filter's tools | +| `BUILD_H5Z_FILTER` | OFF | the HDF5 filter (needs HDF5) | +| `BUILD_MDZ` | OFF | `mdz`, for molecular dynamics of solid materials ([tools/mdz](../tools/mdz/README.md)) | +| `BUILD_PARAVIEW_PLUGIN` | OFF | the ParaView reader (needs ParaView) | +| `BUILD_TESTING` | OFF | the tests; only when SZ3 is the top-level project | +| `SZ3_INSTALL` | ON at top level, OFF in a subproject | install rules | +| `SZ3_USE_BUNDLED_ZSTD` | OFF (ON with MSVC) | use `tools/zstd` even when a system Zstd is found | +| `SZ3_DEBUG_TIMINGS` | OFF | print timings | +| `H5Z_SZ3_PLUGIN_INSTALL_DIR` | `lib/plugin` | where the plugin copy of the filter goes; empty for none | +| `CMAKE_DISABLE_FIND_PACKAGE_OpenMP` (CMake's own) | OFF | ON builds SZ3 without OpenMP | + +On Linux with glibc 2.35 or newer, `GLIBC_TUNABLES=glibc.malloc.hugetlb=1` in the environment backs SZ3's large +buffers with huge pages, which makes compressing and decompressing large fields 10-20% faster. + +### Dependencies + +- **Zstd.** pkg-config first, else the vendored Zstd in `tools/zstd`, built as the private static target `sz3_zstd` + with hidden symbols. Nothing is downloaded at configure time, so SZ3 builds offline. No SZ3 header includes + `zstd.h` (`Lossless_zstd.hpp` declares the four functions it calls), so consumers need no Zstd include directory; + keep it that way. +- **OpenMP** is optional and used when CMake finds it; `find_package(OpenMP)` is not `REQUIRED`. +- **HDF5** (C component) for the filter. Never add an MPI search: the filter calls no MPI function, and a parallel + HDF5 brings its own MPI. +- Do not add a dependency for code nothing builds. + +### What is installed + +| Path | Content | +|---|---| +| `bin/` | `sz3`, `sz3_smoke_test`, `mdz`; DLLs on Windows | +| `include/SZ3/`, `include/SZ3c/`, `include/hdf5_sz3/` | the C++ headers with the generated `version.hpp`, the C API, the filter | +| `lib/` | `libSZ3c`, `libhdf5sz3` | +| `lib/plugin/` | a copy of the filter for `HDF5_PLUGIN_PATH` (shared builds only) | +| `lib/cmake/SZ3/` | the CMake package | + +- `libhdf5sz3` and `libSZ3c` carry `VERSION` and a major.minor `SOVERSION`. Their signatures and `SZ3::Config`'s + layout change only in a new major.minor. +- Both are built with hidden visibility; only `HDF5SZ3_EXPORT` / `SZ3C_API` functions are exported, so another SZ3 + copy in the process cannot take over their calls. +- Keep `INSTALL_RPATH_USE_LINK_PATH` on `hdf5sz3`: without a RUNPATH the installed filter cannot find a conda, Spack + or module HDF5. On Windows the filter DLL is in `bin/`, which must be on `PATH`. +- A static `hdf5sz3` is position independent and defines `HDF5SZ3_STATIC` for its consumers. Its own functions + are hidden in the shared library that links it; HDF5's `H5PLextern.h` makes `H5PLget_plugin_type` and + `H5PLget_plugin_info` exported. +- Install rules use `$`. `bin/sz3_smoke_test` exits 0 on a working install; Spack runs it. + +## Using SZ3 from CMake + +### An installed SZ3 + +```cmake +find_package(SZ3 3.4 REQUIRED) # C++ API +target_link_libraries(app PRIVATE SZ3::SZ3) # or SZ3::SZ3core +find_package(SZ3 3.4 REQUIRED COMPONENTS hdf5sz3) # the HDF5 filter +target_link_libraries(app PRIVATE SZ3::hdf5sz3) +``` + +| Target | What it is | +|---|---| +| `SZ3::SZ3core` | The header-only C++17 library and Zstd. No optional dependencies. | +| `SZ3::SZ3` | `SZ3::SZ3core` plus OpenMP, when SZ3 was built with OpenMP and the consumer's compiler has it. Keep it carrying OpenMP: libpressio relies on it. | +| `SZ3::hdf5sz3` | The HDF5 filter library. Usable from C. Its consumers get no OpenMP flags. | + +- Data compressed through `SZ3::SZ3core` or `SZ3::SZ3` decompresses through either. +- `SZ3` and `hdf5sz3` without the namespace are defined too, so `target_link_libraries(app PRIVATE hdf5sz3)` works + against an installed and an in-tree SZ3 alike. +- `find_package(SZ3 )` accepts the same major version. `libSZ3c` has no CMake target; link it by name. +- `SZ3Config.cmake` never ends the consumer's configure. Every search that can decline runs before any target is + created; when one declines, it sets `SZ3_FOUND` false with a reason and leaves no targets behind, so a project can + fall back to its own copy. No `REQUIRED` searches inside it. +- Never export build-machine paths or build-only targets: wrap such paths in `$` and quote + generator expressions that hold paths. +- If the consumer already has `HDF5::HDF5`, SZ3 does not search HDF5 again. OpenMP is searched per enabled language. + +### As a subproject, or copied into another tree + +```cmake +set(BUILD_H5Z_FILTER ON) # only if you need the filter +FetchContent_Declare(SZ3 GIT_REPOSITORY https://github.com/szcompressor/SZ3.git GIT_TAG v3.4.0) +FetchContent_MakeAvailable(SZ3) +target_link_libraries(app PRIVATE SZ3::hdf5sz3) +``` + +- As a subproject SZ3 installs nothing, builds no tools, sets no cache defaults (`BUILD_SHARED_LIBS`, + `CMAKE_BUILD_TYPE`), and builds neither its tests nor the filter's, even when the parent sets `BUILD_TESTING`. Any + new top-level-only behaviour tests `SZ3_IS_TOP_LEVEL`. +- Depend on a release tag, never on a `master` commit. +- A project that copies SZ3 needs `CMakeLists.txt`, `include/`, `tools/zstd` (without + `lib/decompress/huf_decompress_amd64.S`, which `ZSTD_DISABLE_ASM` leaves out) and, for the filter, + `tools/H5Z-SZ3` without `test/` and `tools/`. Keep the license files. +- The options of the parts left out still appear in the parent's cache, and turning one on then fails. Set them as + normal variables before adding SZ3 (CMP0077), and mark the rest advanced: + + ```cmake + set(SZ3_INSTALL OFF) + set(BUILD_SZ3_BINARY OFF) + set(BUILD_MDZ OFF) + set(BUILD_PARAVIEW_PLUGIN OFF) + # after adding SZ3: + mark_as_advanced(SZ3_DEBUG_TIMINGS SZ3_USE_BUNDLED_ZSTD) + ``` +- Link `SZ3::SZ3core` (and `hdf5sz3`) unless you want SZ3's OpenMP; `SZ3::SZ3` adds `-fopenmp` and libgomp to the + consumer. Put warning suppressions on `SZ3core`. To build the bundled SZ3 without OpenMP, set + `CMAKE_DISABLE_FIND_PACKAGE_OpenMP` before adding it. + +## The C++ API in depth + +- **Output buffers.** To compress into your own buffer, size it with `SZ3::SZ_compress_size_bound`; `SZ_compress` + refuses a smaller one. "Twice the input" is too small for small arrays and for integers. + + ```cpp + std::vector buf(SZ3::SZ_compress_size_bound(conf)); + size_t cmpSize = SZ_compress(conf, data.data(), buf.data(), buf.size()); + ``` +- **Decompressing into your own buffer.** `SZ_decompress(conf, cmpData, cmpSize, decData)` writes every value of the + original data and cannot check the buffer's size. With `decData == nullptr` it allocates with `new[]`. +- **Bounds that are not positive and finite.** An error bound that is 0, negative or not finite compresses losslessly + with every algorithm; a relative bound over data with Inf ends up there too. +- **Integer data** is compressed as floating point (float for 1- and 2-byte types, double for 4- and 8-byte types), + then rounded and clamped back to the type, so every value is within `floor(bound)` of the original. 8-byte integers + beyond Âą2^53 are refused. +- **Recompression.** Data decompressed and compressed again can exceed the bound, except with `ALGO_NOPRED` and + `ALGO_BIOMD` under an absolute bound. HDF5 recompresses a chunk that is written again. +- `include/SZ3/api/sz.hpp` documents the rest of the API. + +## OpenMP + +OpenMP is used only when you ask for it: set `conf.openmp = true`, or `OpenMP = YES` in a configuration file. + +- SZ3 splits the data along its slowest dimension (the first one in `SZ3::Config`, the last one on the `sz3` command + line) into one part per OpenMP thread, at most one per index of that dimension, and compresses the parts separately. + `ALGO_BIOMD` data `{frames, atoms, 3}` is split along the atoms instead, so every part keeps all frames. The number + of parts is stored in the data. +- Decompression is correct with any number of threads: fewer than were used to compress, under `OMP_THREAD_LIMIT`, or + inside your own parallel region. +- A build without OpenMP decompresses data compressed with OpenMP, one part after another, and ignores `conf.openmp` + when compressing. +- SZ3 does not change your program's thread count and prints nothing. The HDF5 filter does not use OpenMP. +- `SZ3::SZ3` carries OpenMP to your target; `SZ3::SZ3core` does not. +- With MinGW, nested OpenMP regions crash in libgomp itself. + +Rules for SZ3's own code: + +- Never call `omp_set_num_threads`; use `num_threads(n)` on the region. +- Hand out chunks with `parallel for`, never by `omp_get_thread_num()`: a region may get fewer threads than asked. +- No exception may leave a parallel region. Store it per chunk and rethrow after the region. +- Index with `size_t`. Respect the stream's `openmp` flag on decompression. +- Test with `OMP_THREAD_LIMIT`, not `OMP_NUM_THREADS`; CI asserts that CMake found OpenMP. Skip nested-region tests + on MinGW. + +## Floating point and compiler flags + +Builds with different floating-point options (FMA, fast math in Clang, icx and MSVC) decompress the same values. + +- GCC's `-ffast-math` / `-Ofast` and x87 floating point (`FLT_EVAL_METHOD > 0`) stop the build with an error; on + 32-bit x86 build with `-msse2 -mfpmath=sse`. +- Clang, icx and MSVC fast modes are made precise inside SZ3's code with `#pragma float_control(precise, on)` in + `SZImpl.hpp`, for code reached through `SZ3/api/sz.hpp`: include `sz.hpp` before any other SZ3 header. +- GCC's `-fassociative-math`, `-funsafe-math-optimizations` and `-ffinite-math-only` on their own are not detected and + not supported. + +Rules for SZ3's own code: + +- The decompressor recomputes the compressor's reconstructed values, so both must round every step the same way. + Every product that feeds a prediction or a reconstruction goes through `SZ3::nofma()` (`def.hpp`), which rounds it + before the add; otherwise a build that fuses `a * b + c` (Apple Silicon, aarch64, `-march=native`, Spack's default) + decodes other builds' data far past the bound. +- `nofma()` uses a `volatile`. Do not switch it to `__builtin_assoc_barrier`: GCC 13 and 14 drop it in vectorized + loops. +- The compressor reconstructs through the decompressor's code (`LinearQuantizer::recover_pred`). +- Check the error in double: checked in `T`, float errors are rounded before the check and can pass the bound. +- Never test for NaN or Inf with floating-point comparisons in code that must hold under `-ffinite-math-only`; GCC + removes them. Compare the bits. +- New codecs keep fusable floating-point arithmetic off the reconstruction path; `ALGO_BIOMD` predicts on an integer + lattice for that reason. +- Verify with `tools/test/crossBuildDecode.sh` (two builds decode each other's streams) and `test_reconstruction`. CI + builds a `-march=x86-64-v3` job and checks that it emits FMA. + +## HDF5 filter (H5Z-SZ3, id 32024) + +[tools/H5Z-SZ3/README.md](../tools/H5Z-SZ3/README.md) covers installing the filter, using it from `h5repack`, h5py +and C, registering it in an application, chunks and other filters, appending, and the rules for the filter's code. + +## Data format + +SZ3 can decompress data compressed by some earlier versions; [CHANGELOG.md](../CHANGELOG.md) says which. + +Rules for SZ3's own code: + +- `SZ3_DATA_VERSION` (CMakeLists.txt) is the version of the compressed format, separate from the program version. Bump + it when, and only when, the bytes a build writes change. CI pins the bytes to the declared version in + `tools/test/data_format_digest.txt`; if your change moves the digest, either the format changed (bump the version, + update the digest and the CHANGELOG) or you broke something. +- `SZ_compress` writes `SZ3_MAGIC_NUMBER` and this build's data version, never the values in the caller's `Config`. +- Refuse data you cannot decode correctly, and say why. +- The same input compresses to the same bytes on every run and every build. Initialise every member that `save()` + writes, zero buffers that go into the output, and never let hash-map order or ties decide anything that reaches the + stream. +- Stored data is little-endian on every host. Use `SZ3::read` / `SZ3::write`, not `memcpy` of host words. +- `size_est()` is an upper bound on what `save()` writes for the current input. One buffer is sized from it and the + writers get no capacity, so an under-report corrupts the heap silently. +- `Config::save()` has a one-byte length prefix; outgrowing it is an error. +- Changing a `concepts::` interface or removing a public header breaks out-of-tree code (libpressio, hdf5plugin, + GROMACS). Check them and say so in the PR. +- The library never prints and never calls `exit()`. To tell the caller something, throw; in the HDF5 filter, refuse + in `set_local` with the reason on the HDF5 error stack. + +## Untrusted input + +- The compressed stream is untrusted. Check every length, count or index read from it against the bytes that remain + before using it to index, allocate or loop; `decode(..., remaining_length)` and `load(c, remaining_length)` charge + what they consume. +- Lossless `decompress(src, srcLen, dst, dstCap)` takes a capacity. +- Bound allocations by the input, not by a count the input declares. +- Subtract budgets through a checked helper. Do not guard stream values with `assert`. Own buffers with RAII. +- Put checks where a symbol is produced, not on every bit, to keep decompression fast. + +## ALGO_BIOMD + +[molecular-dynamics.md](molecular-dynamics.md) is for users. For SZ3's code: + +- `ALGO_BIOMD` takes `{frames, atoms, 3}` (a one-frame chunk becomes `{atoms, 3}`); other shapes throw + `std::invalid_argument`, and a chunk of more values than an `int` counts throws. +- Coordinates go on an integer lattice spanning `|x| < B`, B the smallest power of two above 2^21 times the bound for + float (2^27 for double). Coordinates past B, and NaN or Inf outside trailing fill frames, store the chunk losslessly. + B depends on the bound only, so decompressed data compressed again gives the same values; keep it that way. +- BIOMD and `ALGO_LORENZO_REG` compress the caller's data without a copy because neither modifies it; a change that + writes to the input restores the copy. +- `{1, 1024, 3}` is a 2D case, because `setDims` drops a dimension of 1: tests of the multi-frame path need two frames + or more. Fill-frame tests need values off the quantization grid. + +## Integrating SZ3 into an application + +- Update the bundled SZ3 in the same commit as the code that adapts to it, so every commit builds and passes. +- Choose the algorithm per dataset: `ALGO_BIOMD` for coordinates only, `ALGO_NOPRED` for velocities and forces, + lossless for the rest. Asking for BIOMD on another type is a programming error, not a file error. +- HDF5 filter tests: see the [filter README](../tools/H5Z-SZ3/README.md#writing-a-chunk-again-and-appending). +- Say in the integrating commit which SZ3 data the new version cannot read. + +## Tests and CI + +- A new test must fail on the code it guards; break the guarded code once to see it go red. +- Check determinism without relying on dirty heap memory: build the object over storage filled with two patterns and + compare what `save()` writes. +- The ASan + UBSan job runs ctest and the CLI; it leaves the HDF5 filter out. +- `size_est()` is tested directly: save into a larger buffer and require the cursor to stay within what was declared. +- Installed-package checks build a separate project against the install (`consumeInstalledSZ3.sh`), including one + without the filter; `filterAccessModes.sh` covers `h5repack`, `h5dump`, `h5ls` and the registration shapes. CI also + builds libpressio and GROMACS against SZ3. +- Integration tests check compressed sizes against `compression_baseline.json` within 5% either way; timings are + recorded, not gated. +- `SZ3::uint` is not the global `uint`; MSVC and MinGW have none. +- A failing tool in a pipe fails the check (`set -o pipefail`). +- Headers stay warning-clean under consumers' warning flags and compile on their own under libstdc++ and libc++: + include what you use. +- Format with the repository's `.clang-format`, and leave lines you did not change alone. + +## Commits, PRs and comments + +- PRs are squash-merged; the commit title is the PR title plus `(#N)`, and the body says what changed in behaviour and + why. +- PR titles state the outcome. A description says what went wrong (with a reproducer or a measurement), what changed, + what it costs, whether the format changed, and how it was tested, including what was not run. +- Numbers, not adjectives: ratios, speed as a ratio of times with how it was measured, and the number of cases. +- The CHANGELOG says what changed for users; details belong in the PR. The README is for first-time users. +- Comments state the reason or constraint a reader needs, not the history of the change. diff --git a/docs/molecular-dynamics.md b/docs/molecular-dynamics.md new file mode 100644 index 00000000..1f982b79 --- /dev/null +++ b/docs/molecular-dynamics.md @@ -0,0 +1,71 @@ +# Compressing molecular-dynamics data with SZ3 + +## Which compressor for which data + +| Data | Use | Notes | +|---|---|---| +| Biomolecular trajectories: coordinates `{frames, atoms, 3}` (GROMACS, H5MD) | `ALGO_BIOMD` | Absolute bound, strict. Uses water and bond geometry. | +| Velocities and forces | `ALGO_NOPRED` | Absolute bound. Neighbours and earlier frames do not predict them at usual output intervals. | +| Box, step, time, energies | `ALGO_LOSSLESS`, or HDF5 shuffle + gzip | A few values per frame, needed exactly. | +| MD of solid materials (crystals, metals) | MDZ, [tools/mdz](../tools/mdz/README.md) | Atoms vibrating around lattice sites. | +| Fields on a grid (densities, potentials) | the default `ALGO_INTERP_LORENZO` | | + +## How well ALGO_BIOMD does + +At XTC's default precision, 5e-4 nm, on 101 trajectories (geometric means; 1 / 20 frames per chunk): + +- **Compression ratio.** All-atom with water: 3.81 / 5.05, against 3.31 for XTC; the best other HDF5 filter within + the bound gives 2.33 / 3.18, lossless filters 1.14–1.28. All-atom without water: 3.27 / 5.36, against 3.12 for + XTC. Coarse-grained: 2.32 / 3.40, against 2.39 for XTC. +- **Speed.** Compression takes 1.74 / 1.36 times XTC's time, decompression 1.34 / 1.44 times. Through HDF5 it writes + 439 / 498 MB/s and reads 574 / 572 MB/s on one core. +- **In GROMACS.** H5MD files of all-atom systems are 0.86–0.87 / 0.62–0.66 times the size of XTC files, and writing + them takes 0.09%–0.62% / 0.09%–0.60% of a GPU run (XTC 0.06%–0.43%). +- **Error.** Every coordinate comes back within the bound. + +Every run, the other bounds, the setup and the comparison with other HDF5 filters are in +[#169](https://github.com/szcompressor/SZ3/pull/169). + +## ALGO_BIOMD + +- **Shape.** `{frames, atoms, 3}`; one frame (`{atoms, 3}`) also works. Other shapes throw `std::invalid_argument`. A + single call or HDF5 chunk holds at most 2^31 - 1 values. +- **Bound.** Absolute (`EB_ABS`), in the units of the coordinates. MD trajectories are usually kept at 5e-4 nm (XTC's + default precision) or 5e-3 nm for coarse-grained runs. float and double. +- **Values stored losslessly.** Coordinates go on an integer lattice that covers `|x| < B`, B the smallest power of + two above 2^21 times the bound for float (2^27 for double). A chunk holding a coordinate with `|x| >= B`, or NaN or + Inf outside trailing fill frames (frames at the end of the chunk in which every value is the same), is stored + losslessly, at a ratio of about 1.1 to 1.5 instead of 3 to 6: + + | Bound (nm) | 1e-5 | 5e-5 | 1e-4 | 5e-4 | 1e-3 | 5e-3 | + |---|---|---|---|---|---|---| + | B for float (nm) | 32 | 128 | 256 | 2048 | 4096 | 16384 | + +- **Recompression.** Decompressing and compressing again with the same bound gives the same values, so a chunk HDF5 + writes again stays within the bound. +- **Frames per chunk.** Each chunk is compressed on its own, and BIOMD finds water and bonds on its first frame. More + frames per chunk give a higher ratio (see above). Coarse-grained runs at one frame per chunk come out below XTC: + most beads have no bonds or water to predict from. + +## The HDF5 filter for trajectories + +[tools/H5Z-SZ3/README.md](../tools/H5Z-SZ3/README.md) covers installing and using the filter. For trajectories: + +- **Chunk shape.** An `ALGO_BIOMD` chunk must have 3 as its last dimension and should hold whole frames, + `(k, atoms, 3)`. Set `chunks` yourself; h5py's automatic chunking splits the atoms and xyz, and creating such a + dataset fails. +- **Registering the filter.** An application that links the filter calls `H5Zregister(H5PLget_plugin_info())` when it + opens any file, so that a program that only reads can read SZ3 data. +- **Appending frames.** Set `H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS` and give the dataset a chunk cache that holds a + whole chunk (`H5Pset_chunk_cache`), also when the file is reopened. Otherwise HDF5 compresses a chunk again each + time frames are added to it. The last, partial chunk is stored uncompressed at full chunk size. +- **Other filters.** Add `H5Pset_fletcher32` after SZ3 to detect damaged chunks; never put shuffle before SZ3. + +## Which setting for which GROMACS output + +| Output | SZ3 setting | Compression ratio, 1 / 20 frames per chunk | +|---|---|---| +| positions | `ALGO_BIOMD`, absolute bound (5e-4 nm is XTC's default), chunks of `(frames, atoms, 3)` | 3.81 / 5.05 (all-atom with water) | +| velocities | `ALGO_NOPRED`, absolute bound (for example 1e-3 nm/ps) | 2.90–2.96 / 2.94–2.99 (Zstd 1.07) | +| forces | `ALGO_NOPRED`, absolute bound (for example 1 kJ/mol/nm) | 3.08–3.68 / 3.10–3.70 (Zstd 1.08–1.37) | +| box, step, time, energies | `ALGO_LOSSLESS`, or shuffle + gzip | – | diff --git a/include/SZ3/api/impl/SZAlgoBioMD.hpp b/include/SZ3/api/impl/SZAlgoBioMD.hpp new file mode 100644 index 00000000..b46a7c2d --- /dev/null +++ b/include/SZ3/api/impl/SZAlgoBioMD.hpp @@ -0,0 +1,63 @@ +#ifndef SZ3_SZ_BIOMD_HPP +#define SZ3_SZ_BIOMD_HPP + +#include +#include +#include +#include + +#include "SZ3/compressor/SZGenericCompressor.hpp" +#include "SZ3/decomposition/SZBioMDDecomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/encoder/SegmentedEncoder.hpp" +#include "SZ3/lossless/Lossless_bypass.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Statistic.hpp" + +namespace SZ3 { + +// Data BIOMD does not code: +// * Other shapes, and chunks of more values than an int counts (a stream holds at most one symbol per value): an +// exception. +// * NaN or Inf outside trailing fill frames, and coordinates beyond the lattice: the chunk is stored losslessly. +// Frames appended to a chunk BIOMD coded can bring either; lossless storage keeps the values BIOMD decoded, so a +// rewritten chunk stays within the bound. +template +size_t SZ_compress_bioMD(Config &conf, const T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); + assert(conf.cmprAlgo == ALGO_BIOMD); + calAbsErrorBound(conf, data); + + auto lossless = [&] { + conf.cmprAlgo = ALGO_LOSSLESS; + return Lossless_zstd().compress(reinterpret_cast(data), conf.num * sizeof(T), cmpData, cmpCap); + }; + if (N > 3 || conf.dims[N - 1] != 3) + throw std::invalid_argument("SZ3 ALGO_BIOMD: data must be of shape (atoms, 3) or (frames, atoms, 3)"); + if (conf.num > size_t(std::numeric_limits::max())) + throw std::invalid_argument("SZ3 ALGO_BIOMD: more values than an int counts; compress fewer frames at a time"); + auto sz = + make_compressor_sz_generic(make_decomposition_biomd(conf), + SegmentedEncoder>(biomd::NUM_STREAMS), Lossless_bypass()); + try { + return sz->compress(conf, const_cast(data), cmpData, cmpCap); + } catch (const biomd::Fallback &) { + return lossless(); + } +} + +template +void SZ_decompress_bioMD(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + assert(conf.cmprAlgo == ALGO_BIOMD); + + auto sz = + make_compressor_sz_generic(make_decomposition_biomd(conf), + SegmentedEncoder>(biomd::NUM_STREAMS), Lossless_bypass()); + sz->decompress(conf, cmpData, cmpSize, decData); +} + +} // namespace SZ3 +#endif diff --git a/include/SZ3/api/impl/SZAlgoInterp.hpp b/include/SZ3/api/impl/SZAlgoInterp.hpp new file mode 100644 index 00000000..ac4214b6 --- /dev/null +++ b/include/SZ3/api/impl/SZAlgoInterp.hpp @@ -0,0 +1,293 @@ +#ifndef SZ3_SZALGO_INTERP_HPP +#define SZ3_SZALGO_INTERP_HPP + +#include + +#include "SZ3/api/impl/SZAlgoLorenzoReg.hpp" +#include "SZ3/decomposition/BlockwiseDecomposition.hpp" +#include "SZ3/decomposition/InterpolationDecomposition.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Extraction.hpp" +#include "SZ3/utils/QuantOptimization.hpp" +#include "SZ3/utils/Sample.hpp" +#include "SZ3/utils/Statistic.hpp" + +namespace SZ3 { +template +size_t SZ_compress_Interp(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); + assert(conf.cmprAlgo == ALGO_INTERP); + calAbsErrorBound(conf, data); + if (conf.interpAnchorStride < 0) { // set default anchor stride + std::array anchor_strides = {4096, 128, 32, 16}; + conf.interpAnchorStride = anchor_strides[N - 1]; + } + + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), Lossless_zstd()); + return sz->compress(conf, data, cmpData, cmpCap); +} + +template > +void SZ_decompress_Interp(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + assert(conf.cmprAlgo == ALGO_INTERP); + auto cmpDataPos = cmpData; + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + Encoder(), Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + +template +double interp_compress_test( + const std::vector> sampled_blocks, const Config conf, uchar *cmpData, + size_t cmpCap) { // test interp cmp on a set of sampled data blocks and return the compression ratio + auto sz = + make_decomposition_interpolation(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)); + + std::vector total_quant_bins; + for (size_t k = 0; k < sampled_blocks.size(); k++) { + auto cur_block = sampled_blocks[k]; + auto quant_bins = sz.compress(conf, cur_block.data()); + total_quant_bins.insert(total_quant_bins.end(), quant_bins.begin(), + quant_bins.end()); // merge the quant bins. Lossless them together + } + + auto encoder = HuffmanEncoder(); + auto lossless = Lossless_zstd(); + encoder.preprocess_encode(total_quant_bins, sz.get_out_range().second); + size_t bufferSize = + std::max(1000, 1.2 * (sz.size_est() + encoder.size_est() + sizeof(T) * total_quant_bins.size())); + + // Owned, because the encoder and the lossless layer below can throw. + std::unique_ptr buffer_owner(new uchar[bufferSize]); + uchar *const buffer = buffer_owner.get(); + uchar *buffer_pos = buffer; + sz.save(buffer_pos); + encoder.save(buffer_pos); + + // store the size of quant_inds is necessary as it is not always equal to conf.num + write(total_quant_bins.size(), buffer_pos); + encoder.encode(total_quant_bins, buffer_pos); + encoder.postprocess_encode(); + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + auto compression_ratio = conf.num * sampled_blocks.size() * sizeof(T) * 1.0 / cmpSize; + return compression_ratio; +} + +template +double lorenzo_compress_test( + const std::vector> sampled_blocks, const Config &conf, uchar *cmpData, + size_t cmpCap) { // test lorenzo cmp on a set of sampled data blocks and return the compression ratio + std::vector total_quant_bins; + // if ((N == 3 && !conf.regression2) || (N == 1 && !conf.regression && !conf.regression2)) { + std::vector>> predictors; + predictors.push_back(std::make_shared>(conf.absErrorBound)); + predictors.push_back(std::make_shared>(conf.absErrorBound)); + auto sz = make_decomposition_blockwise(conf, ComposedPredictor(predictors), + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)); + // auto sz = make_decomposition_lorenzo_regression(conf, LinearQuantizer(conf.absErrorBound, + // conf.quantbinCnt / 2)); + for (size_t k = 0; k < sampled_blocks.size(); k++) { + auto cur_block = sampled_blocks[k]; + auto quant_bins = sz.compress(conf, cur_block.data()); + total_quant_bins.insert(total_quant_bins.end(), quant_bins.begin(), + quant_bins.end()); // merge the quant bins. Lossless them together + } + auto encoder = HuffmanEncoder(); + auto lossless = Lossless_zstd(); + encoder.preprocess_encode(total_quant_bins, conf.quantbinCnt); + size_t bufferSize = std::max(1000, 1.2 * (encoder.size_est() + sizeof(T) * total_quant_bins.size())); + + // Owned, because the encoder and the lossless layer below can throw. + std::unique_ptr buffer_owner(new uchar[bufferSize]); + uchar *const buffer = buffer_owner.get(); + uchar *buffer_pos = buffer; + sz.save(buffer_pos); + encoder.save(buffer_pos); + + // store the size of quant_inds is necessary as it is not always equal to conf.num + write(total_quant_bins.size(), buffer_pos); + encoder.encode(total_quant_bins, buffer_pos); + encoder.postprocess_encode(); + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + auto compression_ratio = conf.num * sampled_blocks.size() * sizeof(T) * 1.0 / cmpSize; + return compression_ratio; + // } + // else{ + // return 0.0; + // } +} + +template +size_t SZ_compress_Interp_lorenzo(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(conf.cmprAlgo == ALGO_INTERP_LORENZO); + + // Timer timer(true); + calAbsErrorBound(conf, data); + + if (conf.interpAnchorStride < 0) { // set default anchor stride + std::array anchor_strides = {4096, 128, 32, 16}; + conf.interpAnchorStride = anchor_strides[N - 1]; + } + + std::array sample_Rates = {0.005, 0.005, 0.005, + 0.005}; // default data sample rate. todo: add a config var to control + auto sampleRate = sample_Rates[N - 1]; + std::array sampleBlock_Sizes = {4096, 128, 32, + 16}; // default sampled data block rate. Should better be no smaller + // than the anchor stride. todo: add a config var to control + size_t sampleBlockSize = sampleBlock_Sizes[N - 1]; + size_t shortest_edge = conf.dims[0]; + for (size_t i = 0; i < N; i++) { + shortest_edge = conf.dims[i] < shortest_edge ? conf.dims[i] : shortest_edge; + } + // Automatically adjust sampleblocksize. + while (sampleBlockSize >= shortest_edge) sampleBlockSize /= 2; + while (sampleBlockSize >= 16 && (pow(sampleBlockSize + 1, N) / conf.num) > 1.5 * sampleRate) sampleBlockSize /= 2; + if (sampleBlockSize < 8) sampleBlockSize = 8; + + bool to_tune = pow(sampleBlockSize + 1, N) <= 0.05 * conf.num; // to further revise + for (auto &dim : conf.dims) { + if (dim < sampleBlockSize) { + to_tune = false; + break; + } + } + + if (!to_tune) { // if the sampled data would be too many (currently it is 5% of the input), skip the tuning + conf.cmprAlgo = ALGO_INTERP; + return SZ_compress_Interp(conf, data, cmpData, cmpCap); + } + std::vector> sampled_blocks; + size_t per_block_ele_num = pow(sampleBlockSize + 1, N); + size_t sampling_num; + std::vector> starts; + auto profStride = sampleBlockSize / 4; // larger is faster, smaller is better + profiling_block(data, conf.dims, starts, sampleBlockSize, conf.absErrorBound, + profStride); // filter out the non-constant data blocks + size_t num_filtered_blocks = starts.size(); + bool profiling = num_filtered_blocks * per_block_ele_num >= 0.5 * sampleRate * conf.num; // temp. to refine + // bool profiling = false; + sampleBlocks(data, conf.dims, sampleBlockSize, sampled_blocks, sampleRate, profiling, + starts); // sample out same data blocks + sampling_num = sampled_blocks.size() * per_block_ele_num; + + if (sampling_num == 0 || sampling_num >= conf.num * 0.2) { + conf.cmprAlgo = ALGO_INTERP; + return SZ_compress_Interp(conf, data, cmpData, cmpCap); + } + double best_lorenzo_ratio = 0, best_interp_ratio = 0, ratio; + size_t bufferCap = conf.num * sizeof(T); + // Owned, because the compressions below can throw. + std::unique_ptr buffer_owner(new uchar[bufferCap]); + uchar *const buffer = buffer_owner.get(); + Config lorenzo_config = conf; + + { + // tune interp + conf.interpDirection = 0; + conf.interpAlpha = 1.25; + conf.interpBeta = 2.0; + auto testConfig = conf; + std::vector dims(N, sampleBlockSize + 1); + testConfig.setDims(dims.begin(), dims.end()); + for (auto &interp_op : {INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC}) { + testConfig.interpAlgo = interp_op; + ratio = interp_compress_test(sampled_blocks, testConfig, buffer, bufferCap); + if (ratio > best_interp_ratio) { + best_interp_ratio = ratio; + conf.interpAlgo = interp_op; + } + } + + testConfig.interpAlgo = conf.interpAlgo; + testConfig.interpDirection = factorial(N) - 1; + ratio = interp_compress_test(sampled_blocks, testConfig, buffer, bufferCap); + if (ratio > best_interp_ratio * 1.02) { + best_interp_ratio = ratio; + conf.interpDirection = testConfig.interpDirection; + } + testConfig.interpDirection = conf.interpDirection; + // test more alpha-beta pairs for best compression ratio, + auto alphalist = std::vector{1.0, 1.5, 2.0}; + auto betalist = std::vector{1.0, 2.5, 3.0}; + for (size_t i = 0; i < alphalist.size(); i++) { + auto alpha = alphalist[i]; + auto beta = betalist[i]; + testConfig.interpAlpha = alpha; + testConfig.interpBeta = beta; + ratio = interp_compress_test(sampled_blocks, testConfig, buffer, bufferCap); + if (ratio > best_interp_ratio * 1.02) { + best_interp_ratio = ratio; + conf.interpAlpha = alpha; + conf.interpBeta = beta; + } + } + } + { + // only test lorenzo for 1D + if (N == 1 && best_interp_ratio < 50) { + std::vector sample_dims(N, sampleBlockSize + 1); + lorenzo_config.cmprAlgo = ALGO_LORENZO_REG; + lorenzo_config.setDims(sample_dims.begin(), sample_dims.end()); + lorenzo_config.lorenzo = true; + lorenzo_config.lorenzo2 = true; + lorenzo_config.regression = false; + lorenzo_config.regression2 = false; + lorenzo_config.openmp = false; + lorenzo_config.blockSize = 5; + // lorenzo_config.quantbinCnt = 65536 * 2; + best_lorenzo_ratio = lorenzo_compress_test(sampled_blocks, lorenzo_config, buffer, bufferCap); + // delete[]cmprData; + // printf("Lorenzo ratio = %.2f\n", ratio); + } + } + + bool useInterp = !(best_lorenzo_ratio >= best_interp_ratio * 1.1 && best_lorenzo_ratio < 50 && + best_interp_ratio < 50); // 1.1 is a fix coefficient. subject to revise + size_t cmpSize = 0; + if (useInterp) { + conf.cmprAlgo = ALGO_INTERP; + cmpSize = SZ_compress_Interp(conf, data, cmpData, cmpCap); + } else { + // no need to tune lorenzo for 3D anymore + // if (N == 3) { + // float pred_freq, mean_freq; + // T mean_guess; + // lorenzo_config.quantbinCnt = optimize_quant_invl_3d( + // data, conf.dims[0], conf.dims[1], conf.dims[2], conf.absErrorBound, pred_freq, mean_freq, + // mean_guess); + // lorenzo_config.pred_dim = 2; + // ratio = + // lorenzo_compress_test(sampled_blocks, lorenzo_config, buffer, bufferCap); + // if (ratio > best_lorenzo_ratio * 1.02) { + // best_lorenzo_ratio = ratio; + // } else { + // lorenzo_config.pred_dim = 3; + // } + // } + + if (conf.relErrorBound < 1.01e-6 && best_lorenzo_ratio > 5 && lorenzo_config.quantbinCnt != 16384) { + auto quant_num = lorenzo_config.quantbinCnt; + lorenzo_config.quantbinCnt = 16384; + ratio = lorenzo_compress_test(sampled_blocks, lorenzo_config, buffer, bufferCap); + if (ratio > best_lorenzo_ratio * 1.02) { + best_lorenzo_ratio = ratio; + } else { + lorenzo_config.quantbinCnt = quant_num; + } + } + lorenzo_config.setDims(conf.dims.begin(), conf.dims.end()); + conf = lorenzo_config; + // double tuning_time = timer.stop(); + cmpSize = SZ_compress_LorenzoReg(conf, data, cmpData, cmpCap); + } + + return cmpSize; +} +} // namespace SZ3 +#endif diff --git a/include/SZ3/api/impl/SZAlgoLorenzoReg.hpp b/include/SZ3/api/impl/SZAlgoLorenzoReg.hpp new file mode 100644 index 00000000..5da68afc --- /dev/null +++ b/include/SZ3/api/impl/SZAlgoLorenzoReg.hpp @@ -0,0 +1,88 @@ +#ifndef SZ3_SZALGO_LORENZO_REG_HPP +#define SZ3_SZALGO_LORENZO_REG_HPP + +#include +#include + +#include "SZ3/compressor/SZGenericCompressor.hpp" +#include "SZ3/decomposition/BlockwiseDecomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "SZ3/predictor/ComposedPredictor.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/predictor/RegressionPredictor.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Iterator.hpp" +#include "SZ3/utils/QuantOptimization.hpp" +#include "SZ3/utils/Statistic.hpp" + +namespace SZ3 { +template +std::shared_ptr> make_compressor_lorenzo_regression(const Config &conf, + Quantizer quantizer, + Encoder encoder, + Lossless lossless) { + std::vector>> predictors; + + int methodCnt = (conf.lorenzo + conf.lorenzo2 + conf.regression); + int use_single_predictor = (methodCnt == 1); + if (methodCnt == 0) { + throw std::invalid_argument("All lorenzo and regression methods are disabled."); + } + if (conf.lorenzo) { + if (use_single_predictor) { + return make_compressor_sz_generic( + make_decomposition_blockwise(conf, LorenzoPredictor(conf.absErrorBound), quantizer), + encoder, lossless); + } else { + predictors.push_back(std::make_shared>(conf.absErrorBound)); + } + } + if (conf.lorenzo2) { + if (use_single_predictor) { + return make_compressor_sz_generic( + make_decomposition_blockwise(conf, LorenzoPredictor(conf.absErrorBound), quantizer), + encoder, lossless); + } else { + predictors.push_back(std::make_shared>(conf.absErrorBound)); + } + } + if (conf.regression) { + if (use_single_predictor) { + return make_compressor_sz_generic( + make_decomposition_blockwise( + conf, RegressionPredictor(conf.blockSize, conf.absErrorBound), quantizer), + encoder, lossless); + } else { + predictors.push_back( + std::make_shared>(conf.blockSize, conf.absErrorBound)); + } + } + + return make_compressor_sz_generic( + make_decomposition_blockwise(conf, ComposedPredictor(predictors), quantizer), encoder, + lossless); +} + +template +size_t SZ_compress_LorenzoReg(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); + assert(conf.cmprAlgo == ALGO_LORENZO_REG); + calAbsErrorBound(conf, data); + + auto quantizer = LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2); + auto sz = make_compressor_lorenzo_regression(conf, quantizer, HuffmanEncoder(), Lossless_zstd()); + return sz->compress(conf, data, cmpData, cmpCap); +} + +template > +void SZ_decompress_LorenzoReg(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + assert(conf.cmprAlgo == ALGO_LORENZO_REG); + auto cmpDataPos = cmpData; + LinearQuantizer quantizer; + auto sz = make_compressor_lorenzo_regression(conf, quantizer, Encoder(), Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} +} // namespace SZ3 +#endif diff --git a/include/SZ3/api/impl/SZAlgoNopred.hpp b/include/SZ3/api/impl/SZAlgoNopred.hpp new file mode 100644 index 00000000..cb496058 --- /dev/null +++ b/include/SZ3/api/impl/SZAlgoNopred.hpp @@ -0,0 +1,36 @@ +#ifndef SZ3_SZALGO_NOPRED_HPP +#define SZ3_SZALGO_NOPRED_HPP + +#include "SZ3/compressor/SZGenericCompressor.hpp" +#include "SZ3/decomposition/NoPredictionDecomposition.hpp" +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" +#include "SZ3/utils/Config.hpp" + +namespace SZ3 { +template +size_t SZ_compress_nopred(Config &conf, T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); + assert(conf.cmprAlgo == ALGO_NOPRED); + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_generic( + make_decomposition_noprediction(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), Lossless_zstd()); + return sz->compress(conf, data, cmpData, cmpCap); + // return cmpData; +} + +template +void SZ_decompress_nopred(const Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + assert(conf.cmprAlgo == ALGO_NOPRED); + auto cmpDataPos = cmpData; + auto sz = make_compressor_sz_generic( + make_decomposition_noprediction(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + +} // namespace SZ3 +#endif diff --git a/include/SZ3/api/impl/SZDispatcher.hpp b/include/SZ3/api/impl/SZDispatcher.hpp new file mode 100644 index 00000000..4a0227ef --- /dev/null +++ b/include/SZ3/api/impl/SZDispatcher.hpp @@ -0,0 +1,128 @@ +#ifndef SZ3_IMPL_SZDISPATCHER_HPP +#define SZ3_IMPL_SZDISPATCHER_HPP + +#include +#include +#include + +#include "SZ3/api/impl/SZAlgoBioMD.hpp" +#include "SZ3/api/impl/SZAlgoInterp.hpp" +#include "SZ3/api/impl/SZAlgoLorenzoReg.hpp" +#include "SZ3/api/impl/SZAlgoNopred.hpp" +#include "SZ3/deprecated/HuffmanEncoderV1.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Statistic.hpp" + +namespace SZ3 { +template +size_t SZ_compress_dispatcher(Config &conf, const T *data, uchar *cmpData, size_t cmpCap) { + assert(N == conf.N); + if (conf.cmprAlgo == ALGO_BIOMDXTC) { + throw std::invalid_argument("SZ3: ALGO_BIOMDXTC was removed in 3.4.0; use ALGO_BIOMD"); + } + calAbsErrorBound(conf, data); + size_t cmpSize = 0; + + // a bound that is 0, negative or not finite (a relative one over data with Inf among them): lossless only mode + if (!(conf.absErrorBound > 0) || !std::isfinite(conf.absErrorBound)) { + conf.cmprAlgo = ALGO_LOSSLESS; + } + + // do lossy compression + bool isCmpCapSufficient = true; + if (conf.cmprAlgo != ALGO_LOSSLESS) { + try { + // BIOMD and LORENZO_REG only read their input: LORENZO_REG's blockwise decomposition works on a copy padded + // by at least 2, its fallback Lorenzo's padding + if (conf.cmprAlgo == ALGO_BIOMD) { + return SZ_compress_bioMD(conf, data, cmpData, cmpCap); + } else if (conf.cmprAlgo == ALGO_LORENZO_REG) { + cmpSize = SZ_compress_LorenzoReg(conf, const_cast(data), cmpData, cmpCap); + } else { + // the others write their reconstruction into the data they are given, so they get a copy + std::vector dataCopy(data, data + conf.num); + if (conf.cmprAlgo == ALGO_INTERP) { + cmpSize = SZ_compress_Interp(conf, dataCopy.data(), cmpData, cmpCap); + } else if (conf.cmprAlgo == ALGO_INTERP_LORENZO) { + cmpSize = SZ_compress_Interp_lorenzo(conf, dataCopy.data(), cmpData, cmpCap); + } else if (conf.cmprAlgo == ALGO_NOPRED) { + cmpSize = SZ_compress_nopred(conf, dataCopy.data(), cmpData, cmpCap); + } else { + throw std::invalid_argument("Unknown compression algorithm"); + } + } + + } catch (std::length_error &e) { + if (std::string(e.what()) == SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH) { + isCmpCapSufficient = false; + // printf("SZ is downgraded to lossless mode because the buffer for compressed data is not large enough.\n"); + } else { + throw; + } + } + } + + // do lossless only compression if 1) cmpr algorithm is lossless or 2) compressed buffer not large enough for lossy + if (conf.cmprAlgo == ALGO_LOSSLESS || !isCmpCapSufficient) { + conf.cmprAlgo = ALGO_LOSSLESS; + auto zstd = Lossless_zstd(); + return zstd.compress(reinterpret_cast(data), conf.num * sizeof(T), cmpData, cmpCap); + } + + // if lossy compression ratio < 3, test if lossless only mode has a better ratio than lossy + if (conf.num * sizeof(T) / 1.0 / cmpSize < 3) { + auto zstd = Lossless_zstd(); + // No bigger than the lossy stream: zstd gives up as soon as its output could not be smaller. + std::unique_ptr zstdCmpData(new uchar[cmpSize]); + try { + size_t zstdCmpSize = + zstd.compress(reinterpret_cast(data), conf.num * sizeof(T), zstdCmpData.get(), cmpSize); + if (zstdCmpSize < cmpSize) { + conf.cmprAlgo = ALGO_LOSSLESS; + memcpy(cmpData, zstdCmpData.get(), zstdCmpSize); + cmpSize = zstdCmpSize; + } + } catch (const std::length_error &) { + } + } + return cmpSize; +} + +template +void SZ_decompress_dispatcher(Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + { + // backward compatibility for v3.3.2; its Huffman streams are HuffmanEncoderV1's. + if (conf.sz3DataVer < versionInt("3.4.0")) { + if (conf.cmprAlgo == ALGO_LORENZO_REG) + return SZ_decompress_LorenzoReg>(conf, cmpData, cmpSize, decData); + if (conf.cmprAlgo == ALGO_INTERP) + return SZ_decompress_Interp>(conf, cmpData, cmpSize, decData); + if (conf.cmprAlgo != ALGO_LOSSLESS) + throw std::invalid_argument("SZ3: " + enum_to_string(static_cast(conf.cmprAlgo), ALGO_MAP) + + " data of version " + versionStr(conf.sz3DataVer) + + " is not supported; use SZ3 v" + versionStr(conf.sz3DataVer) + + " to decompress it"); + } + } + if (conf.cmprAlgo == ALGO_LOSSLESS) { + auto zstd = Lossless_zstd(); + auto decDataPos = reinterpret_cast(decData); + size_t decDataSize = zstd.decompress(cmpData, cmpSize, decDataPos, conf.num * sizeof(T)); + if (decDataSize != conf.num * sizeof(T)) { + throw std::runtime_error("Decompressed data size does not match the original data size"); + } + } else if (conf.cmprAlgo == ALGO_LORENZO_REG) { + SZ_decompress_LorenzoReg(conf, cmpData, cmpSize, decData); + } else if (conf.cmprAlgo == ALGO_INTERP) { + SZ_decompress_Interp(conf, cmpData, cmpSize, decData); + } else if (conf.cmprAlgo == ALGO_NOPRED) { + SZ_decompress_nopred(conf, cmpData, cmpSize, decData); + } else if (conf.cmprAlgo == ALGO_BIOMD) { + SZ_decompress_bioMD(conf, cmpData, cmpSize, decData); + } else { + throw std::invalid_argument("Unknown compression algorithm"); + } +} +} // namespace SZ3 +#endif diff --git a/include/SZ3/api/impl/SZImpl.hpp b/include/SZ3/api/impl/SZImpl.hpp new file mode 100644 index 00000000..489b36ce --- /dev/null +++ b/include/SZ3/api/impl/SZImpl.hpp @@ -0,0 +1,93 @@ +#ifndef SZ3_IMPL_SZ_HPP +#define SZ3_IMPL_SZ_HPP + +#include +#include +#include +#include +#include +#include + +// Turns off fast floating-point math in the code included below under Clang (icx too) and MSVC, whatever their +// options; otherwise builds decompress different values. +#if defined(__clang__) || defined(_MSC_VER) +#pragma float_control(precise, on, push) +#endif + +#include "SZ3/api/impl/SZDispatcher.hpp" +#include "SZ3/api/impl/SZImplOMP.hpp" +#include "SZ3/def.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" + +namespace SZ3 { +template +using SZ_float_of = std::conditional_t; + +template +size_t SZ_compress_impl(Config &conf, const T *data, uchar *cmpData, size_t cmpCap) { + // Integers are compressed as SZ_float_of, which holds every value of T (8-byte ones within +-2^53), within + // floor(bound) + 0.49, so rounding each decompressed value lands within the bound. + if constexpr (std::is_integral::value) { + std::vector> values(data, data + conf.num); + if (sizeof(T) == 8 && + std::any_of(values.begin(), values.end(), [](double v) { return std::fabs(v) >= 0x1p53; })) + throw std::invalid_argument("SZ3: 8-byte integers must lie within +-2^53, where doubles hold them exactly"); + calAbsErrorBound(conf, values.data()); + conf.absErrorBound = std::floor(conf.absErrorBound) + 0.49; + return SZ_compress_impl, N>(conf, values.data(), cmpData, cmpCap); + } else { + // Floating-point data is split into OpenMP chunks, or compressed as a whole by the dispatcher. +#ifndef _OPENMP + conf.openmp = false; +#endif + if (conf.openmp) { + return SZ_compress_OMP(conf, data, cmpData, cmpCap); + } else { + return SZ_compress_dispatcher(conf, data, cmpData, cmpCap); + } + } +} + +template +void SZ_decompress_impl(Config &conf, const uchar *cmpData, size_t cmpSize, T *decData) { + // Integers are decompressed as SZ_float_of, then rounded; floating-point data as SZ_compress_impl split it. + if constexpr (std::is_integral::value) { + using F = SZ_float_of; + std::vector values(conf.num); + SZ_decompress_impl(conf, cmpData, cmpSize, values.data()); + const T lo = std::numeric_limits::lowest(), hi = std::numeric_limits::max(); + // Clamped first: converting a value outside T is undefined. A NaN from damaged data becomes lo. + std::transform(values.begin(), values.end(), decData, [&](F v) { + v = std::round(v); + return !(v > F(lo)) ? lo : v >= F(hi) ? hi : static_cast(v); + }); + } else if (conf.openmp) { + SZ_decompress_OMP(conf, cmpData, cmpSize, decData); + } else { + SZ_decompress_dispatcher(conf, cmpData, cmpSize, decData); + } +} + +template +size_t SZ_compress_size_bound(const Config &conf) { + if constexpr (std::is_integral::value) { + return SZ_compress_size_bound>(conf); + } else { + bool omp = conf.openmp; +#ifndef _OPENMP + omp = false; +#endif + if (omp) { + return 4096 + SZ_compress_size_bound_omp(conf); + } else { + return 4096 + conf.size_est() + Lossless_zstd::compress_bound(conf.num * sizeof(T)); + } + } +} + +} // namespace SZ3 + +#if defined(__clang__) || defined(_MSC_VER) +#pragma float_control(pop) +#endif +#endif diff --git a/include/SZ3/api/impl/SZImplOMP.hpp b/include/SZ3/api/impl/SZImplOMP.hpp new file mode 100644 index 00000000..7569e236 --- /dev/null +++ b/include/SZ3/api/impl/SZImplOMP.hpp @@ -0,0 +1,274 @@ +#ifndef SZ3_IMPL_SZDISPATCHER_OMP_HPP +#define SZ3_IMPL_SZDISPATCHER_OMP_HPP + +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/impl/SZDispatcher.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" + +#ifdef _OPENMP + +#include + +#endif +namespace SZ3 { +// Without OpenMP the pragmas below drop out and the same code runs as one thread, so a build +// without OpenMP still reads this chunked layout. Keep the code outside the pragmas. +template +size_t SZ_compress_OMP(Config& conf, const T* data, uchar* cmpData, size_t cmpCap) { + int nThreads = 1; +#ifdef _OPENMP +#pragma omp parallel +#pragma omp single + { + nThreads = omp_get_num_threads(); + } +#endif + // ALGO_BIOMD predicts each frame from the previous one, so {frames, atoms, 3} is split along the atoms and every + // chunk keeps all frames; other data along its first dimension. A chunk is outer runs of (hi - lo) * inner values. + const size_t split = conf.cmprAlgo == ALGO_BIOMD && conf.N == 3; + const size_t ext = conf.dims[split], outer = split ? conf.dims[0] : 1, inner = conf.num / (outer * ext); + if (ext < static_cast(nThreads)) { + nThreads = static_cast(ext); + } + std::vector min_t(nThreads), max_t(nThreads); + std::vector conf_t(nThreads); + std::vector> compressed_t(nThreads); + std::vector cmp_size_t(nThreads), cmp_start_t(nThreads + 1); + + // Each loop below hands out chunks, not thread ids: a region may get fewer threads than asked + // for (nested parallelism, OMP_THREAD_LIMIT, dynamic adjustment), and every chunk still runs. + if (conf.errorBoundMode != EB_ABS) { +#ifdef _OPENMP +#pragma omp parallel for num_threads(nThreads) +#endif + for (int i = 0; i < nThreads; i++) { + size_t lo = static_cast(i) * conf.num / nThreads; + size_t hi = static_cast(i + 1) * conf.num / nThreads; + auto minmax = std::minmax_element(data + lo, data + hi); + min_t[i] = *minmax.first; + max_t[i] = *minmax.second; + } + T range = *std::max_element(max_t.begin(), max_t.end()) - *std::min_element(min_t.begin(), min_t.end()); + calAbsErrorBound(conf, data, range); + } + + // An exception that leaves an OpenMP region does not unwind to the caller -- the runtime calls + // std::terminate -- so each chunk stores its own and the first one is rethrown below. + std::exception_ptr failure; + // num_threads(nThreads) applies to this region alone. omp_set_num_threads(nThreads) would write + // the process-wide nthreads-var instead, and that outlives the call: an application running 32 + // threads that compresses one chunk here would go on running 8 afterwards. SZ3 is a library + // inside someone else's program and has no business changing that. +#ifdef _OPENMP +#pragma omp parallel for num_threads(nThreads) +#endif + for (int i = 0; i < nThreads; i++) try { + size_t lo = static_cast(i) * ext / nThreads; + size_t hi = static_cast(i + 1) * ext / nThreads; + auto dims_t = conf.dims; + dims_t[split] = hi - lo; + conf_t[i] = conf; + conf_t[i].setDims(dims_t.begin(), dims_t.end()); + // Room for the size header Lossless_zstd::compress writes ahead of the zstd stream. + size_t cmp_size_cap = sizeof(uint64_t) + Lossless_zstd::compress_bound(conf_t[i].num * sizeof(T)); + compressed_t[i].reset(new uchar[cmp_size_cap]); + const T* data_t = data + lo * inner; + std::vector gathered; + if (outer > 1) { + gathered.resize(conf_t[i].num); + const size_t run = (hi - lo) * inner; + for (size_t o = 0; o < outer; o++) std::copy_n(data + (o * ext + lo) * inner, run, &gathered[o * run]); + data_t = gathered.data(); + } + // we have to use conf_t[i].N instead of N since each chunk may be a slice of the original data + if (conf_t[i].N == 1) { + cmp_size_t[i] = SZ_compress_dispatcher(conf_t[i], data_t, compressed_t[i].get(), cmp_size_cap); + } else if (conf_t[i].N == 2) { + cmp_size_t[i] = SZ_compress_dispatcher(conf_t[i], data_t, compressed_t[i].get(), cmp_size_cap); + } else if (conf_t[i].N == 3) { + cmp_size_t[i] = SZ_compress_dispatcher(conf_t[i], data_t, compressed_t[i].get(), cmp_size_cap); + } else if (conf_t[i].N == 4) { + cmp_size_t[i] = SZ_compress_dispatcher(conf_t[i], data_t, compressed_t[i].get(), cmp_size_cap); + } else { + throw std::invalid_argument("Unsupported N"); + } + } catch (...) { +#ifdef _OPENMP +#pragma omp critical +#endif + { + if (!failure) failure = std::current_exception(); + } + } + if (failure) { + std::rethrow_exception(failure); + } + + size_t header_size = sizeof(int) + nThreads * sizeof(uint64_t); + cmp_start_t[0] = 0; + for (int i = 0; i < nThreads; i++) { + header_size += conf_t[i].size_est(); + cmp_start_t[i + 1] = cmp_start_t[i] + cmp_size_t[i]; + } + // cmpCap may come from a bound for fewer chunks, computed by a copy of this header built without OpenMP. + if (header_size + cmp_start_t[nThreads] > cmpCap) { + throw std::invalid_argument(SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH); + } + uchar* buffer_pos = cmpData; + write(nThreads, buffer_pos); + for (int i = 0; i < nThreads; i++) { + conf_t[i].save(buffer_pos); + } + for (int i = 0; i < nThreads; i++) write(cmp_size_t[i], buffer_pos); +#ifdef _OPENMP +#pragma omp parallel for num_threads(nThreads) +#endif + for (int i = 0; i < nThreads; i++) { + memcpy(buffer_pos + cmp_start_t[i], compressed_t[i].get(), cmp_size_t[i]); + } + return buffer_pos - cmpData + cmp_start_t[nThreads]; +} + +template +void SZ_decompress_OMP(Config& conf, const uchar* cmpData, size_t cmpSize, T* decData) { + auto cmpr_data_pos = cmpData; + const uchar* const cmp_end = cmpData + cmpSize; + int nThreads = 1; + if (static_cast(cmp_end - cmpr_data_pos) < sizeof(nThreads)) + throw std::out_of_range("SZ3 OMP: truncated thread count"); + read(nThreads, cmpr_data_pos); + // Each thread contributes at least a config and a size, so the count cannot exceed the buffer size. + if (nThreads <= 0 || static_cast(nThreads) > cmpSize) + throw std::out_of_range("SZ3 OMP: invalid thread count"); + + // Split as SZ_compress_OMP split it, from the same Config. + const size_t split = conf.cmprAlgo == ALGO_BIOMD && conf.N == 3; + const size_t ext = conf.dims[split], outer = split ? conf.dims[0] : 1, inner = conf.num / (outer * ext); + + std::vector conf_t(nThreads); + for (int i = 0; i < nThreads; i++) { + size_t confRemaining = static_cast(cmp_end - cmpr_data_pos); + conf_t[i].load(cmpr_data_pos, confRemaining); + } + + if (conf_t[0].sz3MagicNumber != SZ3_MAGIC_NUMBER) { + throw std::invalid_argument("magic number mismatch, the input data is not compressed by SZ3"); + } + { + // backward compatibility for v3.3.2; Config::load() does not read the version the dispatcher checks. + for (auto& c : conf_t) c.sz3DataVer = conf.sz3DataVer; + } + + std::vector cmp_start_t, cmp_size_t; + cmp_size_t.resize(nThreads); + if (static_cast(cmp_end - cmpr_data_pos) < static_cast(nThreads) * sizeof(uint64_t)) + throw std::out_of_range("SZ3 OMP: truncated per-thread sizes"); + for (auto& s : cmp_size_t) { + uint64_t v = 0; + read(v, cmpr_data_pos); + s = v; + } + auto cmpr_data_p = cmpr_data_pos; + + cmp_start_t.resize(nThreads + 1); + cmp_start_t[0] = 0; + // The payloads follow back-to-back; build the offsets without overflowing so no slice points out. + const size_t payload_avail = static_cast(cmp_end - cmpr_data_p); + for (int i = 1; i <= nThreads; i++) { + if (cmp_size_t[i - 1] > payload_avail - cmp_start_t[i - 1]) + throw std::out_of_range("SZ3 OMP: per-thread compressed sizes exceed the buffer"); + cmp_start_t[i] = cmp_start_t[i - 1] + cmp_size_t[i - 1]; + } + + std::exception_ptr failure; +#ifdef _OPENMP +#pragma omp parallel num_threads(nThreads) +#endif + try { + // nThreads is how many chunks the writer split the data into -- it came out of the stream, + // not from this machine. num_threads asks for that many threads but nothing guarantees + // them: OMP_THREAD_LIMIT caps the whole program, dynamic adjustment may lower it, and a + // region nested inside the caller's own gets exactly one thread unless nested parallelism + // was turned on, which it is not by default -- the case a filter called from inside an + // application's parallel region lands in. Indexing the chunks by thread id would then skip + // every chunk whose id no thread has, leaving that span of the output untouched and + // reporting success, so each thread walks the chunks in strides of however many arrived. +#ifdef _OPENMP + const int actual = omp_get_num_threads(); + const int first = omp_get_thread_num(); +#else + const int actual = 1; + const int first = 0; +#endif + for (int tid = first; tid < nThreads; tid += actual) { + size_t lo = static_cast(tid) * ext / nThreads; + size_t hi = static_cast(tid + 1) * ext / nThreads; + const size_t run = (hi - lo) * inner; + // A chunk's own Config decides how much is written into its share of decData. + if (conf_t[tid].num != outer * run) { + throw std::invalid_argument("SZ3: a chunk of the OpenMP stream does not match its share of the data"); + } + T* dec_t = decData + lo * inner; + std::vector gathered; + if (outer > 1) { + gathered.resize(conf_t[tid].num); + dec_t = gathered.data(); + } + if (conf_t[tid].N == 1) { + SZ_decompress_dispatcher(conf_t[tid], cmpr_data_p + cmp_start_t[tid], cmp_size_t[tid], dec_t); + } else if (conf_t[tid].N == 2) { + SZ_decompress_dispatcher(conf_t[tid], cmpr_data_p + cmp_start_t[tid], cmp_size_t[tid], dec_t); + } else if (conf_t[tid].N == 3) { + SZ_decompress_dispatcher(conf_t[tid], cmpr_data_p + cmp_start_t[tid], cmp_size_t[tid], dec_t); + } else if (conf_t[tid].N == 4) { + SZ_decompress_dispatcher(conf_t[tid], cmpr_data_p + cmp_start_t[tid], cmp_size_t[tid], dec_t); + } else { + throw std::invalid_argument("Unsupported N"); + } + if (outer > 1) { + for (size_t o = 0; o < outer; o++) + std::copy_n(&gathered[o * run], run, decData + (o * ext + lo) * inner); + } + } + } catch (...) { +#ifdef _OPENMP +#pragma omp critical +#endif + { + if (!failure) failure = std::current_exception(); + } + } + if (failure) { + std::rethrow_exception(failure); + } +} + +template +size_t SZ_compress_size_bound_omp(const Config& conf) { + // A region without num_threads has at most omp_get_max_threads() threads, so this bounds the + // chunk count SZ_compress_OMP picks, however many threads its region then gets. + int nThreads = 1; +#ifdef _OPENMP + nThreads = omp_get_max_threads(); +#endif + const size_t ext = conf.dims[conf.cmprAlgo == ALGO_BIOMD && conf.N == 3]; + if (ext < static_cast(nThreads)) { + nThreads = static_cast(ext); + } + // the largest chunk: ceil(ext / nThreads) indices of the split dimension + size_t chunk_size = (ext + nThreads - 1) / nThreads * (conf.num / ext); + // for each thread, we save conf, compressed size, and compressed data + // the per-chunk compressed data may carry the size header written by Lossless_zstd::compress + return sizeof(int) + nThreads * conf.size_est() + 2 * nThreads * sizeof(uint64_t) + + nThreads * Lossless_zstd::compress_bound(chunk_size * sizeof(T)); +} +} // namespace SZ3 + +#endif diff --git a/include/SZ3/api/sz.hpp b/include/SZ3/api/sz.hpp new file mode 100644 index 00000000..b6449620 --- /dev/null +++ b/include/SZ3/api/sz.hpp @@ -0,0 +1,204 @@ +/** + * @file sz.hpp + * @brief SZ3 compression and decompression API. + * + * This header provides the main API functions for compressing and decompressing data using SZ3. + * + * Compressed Data Format of SZ3: + * The compressed data is always stored in little-endian order. + * The compressed data consists of three main sections: + * 1. Header (16 bytes): Contains metadata about the compressed data. + * - Magic Number (4 bytes): Identifies the data as SZ3-compressed. + * - Version (4 bytes): Indicates the version of the SZ3 format. + * - Compressed Size (8 bytes): Specifies the size of the compressed payload. + * 2. Compressed Payload: The actual compressed data. + * 3. Configuration: Stores the compression configuration used. + * + * The layout can be visualized as follows: + * [ Header (16 bytes) | Compressed Payload | Configuration ] + */ + +#ifndef SZ3_SZ_HPP +#define SZ3_SZ_HPP + +#include +#include + +#include "SZ3/api/impl/SZImpl.hpp" +#include "SZ3/version.hpp" + +/** + * Compresses the input data using the provided configuration and stores the result in a pre-allocated buffer. + * @tparam T The data type of the source data. + * @param config The compression configuration. + * @param data Pointer to the source data array. + * @param cmpData Pointer to the pre-allocated buffer for compressed data. + * @param cmpCap The size of the pre-allocated buffer in bytes. + * @return The size of the compressed data in bytes. + * @example + * SZ3::Config conf(100, 200, 300); // 300 is the fastest dimension + * conf.errorBoundMode = SZ3::EB_ABS; // Refer to def.hpp for supported error bound modes + * conf.absErrorBound = 1E-3; // Absolute error bound of 1e-3 + * size_t outSize = SZ_compress(conf, data, outBuff, outBuffCap); + */ +template +size_t SZ_compress(const SZ3::Config& config, const T* data, char* cmpData, size_t cmpCap) { + using namespace SZ3; + Config conf(config); + + if (cmpCap < SZ_compress_size_bound(conf)) { + throw std::invalid_argument(SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH); + } + + auto cmpDataPos = reinterpret_cast(cmpData); + + // save 16 bytes header + write(static_cast(SZ3_MAGIC_NUMBER), cmpDataPos); // magic number (4 bytes) + write(versionInt(SZ3_DATA_VER), cmpDataPos); // data version (4 bytes) + auto sizeHeaderPos = cmpDataPos; + cmpDataPos += 8; // reserve space for cmp data size (8 bytes) + + // begin compression + auto cmpDataCap = cmpCap - 16 - conf.size_est() * 2; + uint64_t cmpDataSize = 0; + if (conf.N == 1) { + cmpDataSize = SZ_compress_impl(conf, data, cmpDataPos, cmpDataCap); + } else if (conf.N == 2) { + cmpDataSize = SZ_compress_impl(conf, data, cmpDataPos, cmpDataCap); + } else if (conf.N == 3) { + cmpDataSize = SZ_compress_impl(conf, data, cmpDataPos, cmpDataCap); + } else if (conf.N == 4) { + cmpDataSize = SZ_compress_impl(conf, data, cmpDataPos, cmpDataCap); + } else { + throw std::invalid_argument("Data dimension higher than 4 is not supported."); + } + + // save compressed data size back in header + write(cmpDataSize, sizeHeaderPos); + + // save config + cmpDataPos += cmpDataSize; + auto confSize = conf.save(cmpDataPos); + + return 16 + cmpDataSize + confSize; +} + +/** + * Compresses the input data using the provided configuration and returns a newly allocated buffer containing the compressed data. + * @tparam T The data type of the source data. + * @param config The compression configuration. + * @param data Pointer to the source data array. + * @param cmpSize Output parameter set to the size of the compressed data in bytes. + * @return Pointer to the newly allocated buffer containing the compressed data. The caller is responsible for deleting this buffer using 'delete[]'. + * @note This function allocates memory for the compressed data. Ensure to free it when no longer needed. + */ +template +char* SZ_compress(const SZ3::Config& config, const T* data, size_t& cmpSize) { + using namespace SZ3; + + size_t bufferLen = SZ_compress_size_bound(config); + std::unique_ptr buffer(new char[bufferLen]); + cmpSize = SZ_compress(config, data, buffer.get(), bufferLen); + + return buffer.release(); +} + +/** + * Decompresses the compressed data into a pre-allocated buffer using the configuration loaded from the compressed data. + * @tparam T The data type of the decompressed data. + * @param config Configuration placeholder that will be overwritten with the compression configuration from the compressed data. + * @param cmpData Pointer to the compressed data. + * @param cmpSize The size of the compressed data in bytes. + * @param decData Reference to a pointer for the pre-allocated buffer for decompressed data. If null, a new buffer is allocated. + * @example + * auto decData = new float[100 * 200 * 300]; + * SZ3::Config conf; + * SZ_decompress(conf, cmpData, cmpSize, decData); + */ +template +void SZ_decompress(SZ3::Config& config, const char* cmpData, size_t cmpSize, T*& decData) { + using namespace SZ3; + + auto cmpDataPos = reinterpret_cast(cmpData); + + // Header layout: magic number (4) + data version (4) + compressed payload size (8) = 16 bytes. + if (cmpSize < 16) { + throw std::out_of_range("SZ3: compressed data is smaller than the header"); + } + + read(config.sz3MagicNumber, cmpDataPos); + if (config.sz3MagicNumber != SZ3_MAGIC_NUMBER) { + throw std::invalid_argument("magic number mismatch, the input data is not compressed by SZ3"); + } + + read(config.sz3DataVer, cmpDataPos); + if (config.sz3DataVer < versionInt(SZ3_DATA_VER_OLDEST) || config.sz3DataVer > versionInt(SZ3_DATA_VER)) { + // Do not print here. In the HDF5 filter stdout is the user's output file. + std::stringstream ss; + ss << "SZ3 " << SZ3_VER << " reads data versions " << SZ3_DATA_VER_OLDEST << " to " << SZ3_DATA_VER + << ", but this data is version " << versionStr(config.sz3DataVer) << ". "; + if (config.sz3DataVer > versionInt(SZ3_DATA_VER)) + ss << "Upgrade SZ3 to decompress it."; + else + ss << "Use SZ3 v" << versionStr(config.sz3DataVer) << " to decompress it."; + throw std::invalid_argument(ss.str()); + } + + uint64_t cmpDataSize = 0; + read(cmpDataSize, cmpDataPos); + + // The compressed payload is followed by the serialized config; both must fit in the remaining bytes. + if (cmpDataSize > cmpSize - 16) { + throw std::out_of_range("SZ3: compressed payload size exceeds the buffer"); + } + auto cmpConfPos = cmpDataPos + cmpDataSize; + size_t confRemaining = cmpSize - 16 - cmpDataSize; + config.load(cmpConfPos, confRemaining); + + if (decData == nullptr) { + decData = new T[config.num]; + } + if (config.N == 1) { + SZ_decompress_impl(config, cmpDataPos, cmpDataSize, decData); + } else if (config.N == 2) { + SZ_decompress_impl(config, cmpDataPos, cmpDataSize, decData); + } else if (config.N == 3) { + SZ_decompress_impl(config, cmpDataPos, cmpDataSize, decData); + } else if (config.N == 4) { + SZ_decompress_impl(config, cmpDataPos, cmpDataSize, decData); + } else { + throw std::invalid_argument("Data dimension higher than 4 is not supported."); + } +} + +/** + * Decompresses the compressed data into a buffer it allocates itself, using the configuration loaded from the + * compressed data. Pass a buffer to the four-argument overload instead to decompress into memory you already hold. + * @tparam T The data type of the decompressed data. + * @param config Configuration placeholder that will be overwritten with the compression configuration from the compressed data. + * @param cmpData Pointer to the compressed data. + * @param cmpSize The size of the compressed data in bytes. + * @return A new[] buffer of config.num elements, which the caller owns and must delete[]. There is no null return to + * test: the four-argument overload throws, so a value arriving here is always a buffer. + * @example + * SZ3::Config conf; + * float* decData = SZ_decompress(conf, cmpData, cmpSize); + * // conf now describes the data; conf.num is the element count of decData + * delete[] decData; + */ +template +T* SZ_decompress(SZ3::Config& config, const char* cmpData, size_t cmpSize) { + using namespace SZ3; + T* decData = nullptr; + try { + SZ_decompress(config, cmpData, cmpSize, decData); + } catch (...) { + // The four-argument overload allocates through the reference and can throw afterwards, and + // the caller never sees the pointer when it does. + delete[] decData; + throw; + } + return decData; +} + +#endif diff --git a/include/SZ3/compressor/Compressor.hpp b/include/SZ3/compressor/Compressor.hpp new file mode 100644 index 00000000..a65b564c --- /dev/null +++ b/include/SZ3/compressor/Compressor.hpp @@ -0,0 +1,40 @@ +#ifndef SZ3_COMPRESSOR_HPP +#define SZ3_COMPRESSOR_HPP + +#include "SZ3/def.hpp" +#include "SZ3/utils/Config.hpp" + +namespace SZ3::concepts { + +/** + * Compressor describes the whole compression workflow. + * Compressor is implemented using modules (decomposition or predictor, encoder, lossless, etc.). + * @tparam T input data type + */ +template +class CompressorInterface { + public: + virtual ~CompressorInterface() = default; + + /** + * decompress data + * @param cmpData compressed data in bytes + * @param cmpSize size of compressed data in bytes + * @param decData pre-allocated space to store the decompress data + * @return pointer to the decompress data + */ + virtual T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) = 0; + + /** + * compress data + * @param conf compression configuration + * @param data input data in original format + * @param cmpData compressed data in bytes + * @param cmpCap size of the compressed buffer in bytes + * @return size of the compressed data in bytes + */ + virtual size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) = 0; +}; +} // namespace SZ3::concepts + +#endif diff --git a/include/SZ3/compressor/SZGenericCompressor.hpp b/include/SZ3/compressor/SZGenericCompressor.hpp new file mode 100644 index 00000000..ac98fe7a --- /dev/null +++ b/include/SZ3/compressor/SZGenericCompressor.hpp @@ -0,0 +1,109 @@ +#ifndef SZ3_COMPRESSOR_TYPE_ONE_HPP +#define SZ3_COMPRESSOR_TYPE_ONE_HPP + +#include +#include +#include +#include +#include + +#include "SZ3/compressor/Compressor.hpp" +#include "SZ3/decomposition/Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Timer.hpp" + +namespace SZ3 { + +/** + * SZGenericCompressor glues together decomposition, encoder, and lossless modules to form the compressor. + * It only takes Decomposition, not Predictor. + * @tparam T original data type + * @tparam N original data dimension + * @tparam Decomposition decomposition module + * @tparam Encoder encoder module + * @tparam Lossless lossless module + */ +template +class SZGenericCompressor : public concepts::CompressorInterface { + public: + SZGenericCompressor(Decomposition decomposition_, Encoder encoder_, Lossless lossless_) + : decomposition(decomposition_), encoder(encoder_), lossless(lossless_) { + static_assert(std::is_base_of, Decomposition>::value, + "must implement the frontend interface"); + static_assert(std::is_base_of, Encoder>::value, + "must implement the encoder interface"); + static_assert(std::is_base_of::value, + "must implement the lossless interface"); + } + + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) override { + std::vector quant_inds = decomposition.compress(conf, data); + + if (decomposition.get_out_range().first != 0) { + throw std::runtime_error("The output range of the decomposition must start from 0 for this compressor"); + } + encoder.preprocess_encode(quant_inds, decomposition.get_out_range().second); + size_t bufferSize = std::max( + 1000, 2 * (decomposition.size_est() + encoder.size_est() + sizeof(int) * quant_inds.size())); + + // Owned, because the encoder and the lossless layer below can throw. + std::unique_ptr buffer_owner(new uchar[bufferSize]); + uchar *const buffer = buffer_owner.get(); + uchar *buffer_pos = buffer; + + decomposition.save(buffer_pos); + encoder.save(buffer_pos); + + // store the size of quant_inds is necessary as it is not always equal to conf.num + write(quant_inds.size(), buffer_pos); + encoder.encode(quant_inds, buffer_pos); + encoder.postprocess_encode(); + + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + + return cmpSize; + } + + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *decData) override { + uchar *buffer = nullptr; + size_t bufferSize = lossless.decompress(cmpData, cmpSize, buffer, 0); + + // malloc'd by the lossless layer, hence free(). Owned, because the parsing below can throw. + std::unique_ptr buffer_owner(buffer, &free); + + uchar const *bufferPos = buffer; + + decomposition.load(bufferPos, bufferSize); + encoder.load(bufferPos, bufferSize); + + uint64_t quant_inds_size = 0; + read(quant_inds_size, bufferPos, bufferSize); + auto quant_inds = encoder.decode(bufferPos, quant_inds_size, bufferSize); + encoder.postprocess_decode(); + + // The remaining work uses `quant_inds` and `decData` only, so release the internal buffer now. + buffer_owner.reset(); + + decomposition.decompress(conf, quant_inds, decData); + return decData; + } + + private: + Decomposition decomposition; + Encoder encoder; + Lossless lossless; +}; + +template +std::shared_ptr> make_compressor_sz_generic( + Decomposition decomposition, Encoder encoder, Lossless lossless) { + return std::make_shared>(decomposition, encoder, + lossless); +} + +} // namespace SZ3 +#endif diff --git a/include/SZ3/compressor/specialized/SZExaaltCompressor.hpp b/include/SZ3/compressor/specialized/SZExaaltCompressor.hpp new file mode 100644 index 00000000..f96d88fc --- /dev/null +++ b/include/SZ3/compressor/specialized/SZExaaltCompressor.hpp @@ -0,0 +1,218 @@ +#ifndef SZ3_EXAALT_COMPRESSSOR_HPP +#define SZ3_EXAALT_COMPRESSSOR_HPP + +#include +#include +#include +#include + +#include "SZ3/compressor/Compressor.hpp" +#include "SZ3/def.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/Iterator.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +/** + * specialized compressor for solid-state molecular dynamics simulation data + * it utilizes the data character of levels to boost compression ratio + * see ICDE'22 MDZ paper for details + */ +namespace SZ3 { +template +class SZExaaltCompressor : public SZ3::concepts::CompressorInterface { + public: + SZExaaltCompressor(Quantizer quantizer, Encoder encoder, Lossless lossless, int timestep_op) + : quantizer(quantizer), encoder(encoder), lossless(lossless), timestep_op(timestep_op) { + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quatizer interface"); + static_assert(std::is_base_of, Encoder>::value, + "must implement the encoder interface"); + static_assert(std::is_base_of::value, + "must implement the lossless interface"); + } + + // compress given the error bound + // uchar *compress(T *data, size_t &compressed_size) { + size_t compress(const Config &conf, T *data, uchar *cmpData, size_t cmpCap) override { + assert(!(timestep_op > 0 && conf.dims.size() != 2) && "timestep prediction requires 2d dataset"); + + std::vector quant_inds(conf.num); + std::vector pred_inds(conf.num); + quantizer.precompress_data(); + + // Timer timer(true); + + auto l0 = quantize_to_level(data[0]); + pred_inds[0] = l0 + level_num; + quant_inds[0] = quantizer.quantize_and_overwrite(data[0], level(l0)); + + if (timestep_op == 0) { + for (size_t i = 1; i < conf.num; i++) { + auto l = quantize_to_level(data[i]); + pred_inds[i] = l - l0 + level_num; + quant_inds[i] = quantizer.quantize_and_overwrite(data[i], level(l)); + l0 = l; + } + } else { + std::vector levels(conf.dims[1]); + levels[0] = l0; + for (size_t i = 1; i < conf.dims[1]; i++) { + levels[i] = quantize_to_level(data[i]); + pred_inds[i] = levels[i] - levels[i - 1] + level_num; + quant_inds[i] = quantizer.quantize_and_overwrite(data[i], level(levels[i])); + } + auto pred_idx = conf.dims[1]; + if (timestep_op == 1) { + for (size_t i = 0; i < conf.dims[1]; i++) { + for (size_t t = 1; t < conf.dims[0]; t++) { + size_t idx = t * conf.dims[1] + i; + quant_inds[pred_idx++] = + quantizer.quantize_and_overwrite(data[idx], data[(t - 1) * conf.dims[1] + i]); + } + } + pred_inds.resize(conf.dims[1]); + } else { + for (size_t i = 0; i < conf.dims[1]; i++) { + l0 = levels[i]; + for (size_t t = 1; t < conf.dims[0]; t++) { + size_t idx = t * conf.dims[1] + i; + auto l = quantize_to_level(data[idx]); + pred_inds[pred_idx] = l - l0 + level_num; + quant_inds[pred_idx++] = quantizer.quantize_and_overwrite(data[idx], level(l)); + l0 = l; + } + } + } + assert(pred_idx == conf.num); + } + + // timer.stop("Predition & Quantization"); + + quantizer.postcompress_data(); + + auto buffer = static_cast(malloc(4 * conf.num * sizeof(T))); + uchar *buffer_pos = buffer; + quantizer.save(buffer_pos); + // quantizer.print(); + + + if (quantizer.get_out_range().first != 0) { + throw std::runtime_error("The output range of the quantizer must start from 0 for this compressor"); + } + encoder.preprocess_encode(quant_inds, quantizer.get_out_range().second); + encoder.save(buffer_pos); + encoder.encode(quant_inds, buffer_pos); + encoder.postprocess_encode(); + + // std::cout << *std::min_element(pred_inds.begin(), pred_inds.end()) << std::endl; + // std::cout << *std::max_element(pred_inds.begin(), pred_inds.end()) << std::endl; + + encoder.preprocess_encode(pred_inds, level_num * 2 + 1); + encoder.save(buffer_pos); + encoder.encode(pred_inds, buffer_pos); + encoder.postprocess_encode(); + + auto cmpSize = lossless.compress(buffer, buffer_pos - buffer, cmpData, cmpCap); + free(buffer); + return cmpSize; + } + + // T *decompress(uchar const *lossless_compressed_data, const size_t length) { + T *decompress(const Config &conf, uchar const *cmpData, size_t cmpSize, T *dec_data) override { + uchar *buffer = nullptr; + size_t bufferSize = lossless.decompress(cmpData, cmpSize, buffer, 0); + // The parsing below walks the decompressed buffer, so bufferSize is its bound, not cmpSize. + size_t remaining_length = bufferSize; + uchar const *buffer_pos = buffer; + + quantizer.load(buffer_pos, remaining_length); + encoder.load(buffer_pos, remaining_length); + auto quant_inds = encoder.decode(buffer_pos, conf.num, remaining_length); + encoder.postprocess_decode(); + + encoder.load(buffer_pos, remaining_length); + auto pred_inds_num = (timestep_op == 1) ? conf.dims[1] : conf.num; + auto pred_inds = encoder.decode(buffer_pos, pred_inds_num, remaining_length); + encoder.postprocess_decode(); + + free(buffer); + + quantizer.predecompress_data(); + + auto l = pred_inds[0] - level_num; + dec_data[0] = quantizer.recover(level(l), quant_inds[0]); + + if (timestep_op == 0) { + for (size_t i = 1; i < conf.num; i++) { + l += pred_inds[i] - level_num; + dec_data[i] = quantizer.recover(level(l), quant_inds[i]); + } + } else { + std::vector levels(conf.dims[1]); + levels[0] = l; + for (size_t i = 1; i < conf.dims[1]; i++) { + l += pred_inds[i] - level_num; + dec_data[i] = quantizer.recover(level(l), quant_inds[i]); + levels[i] = l; + } + auto pred_idx = conf.dims[1]; + if (timestep_op == 1) { + for (size_t i = 0; i < conf.dims[1]; i++) { + for (size_t t = 1; t < conf.dims[0]; t++) { + dec_data[t * conf.dims[1] + i] = + quantizer.recover(dec_data[(t - 1) * conf.dims[1] + i], quant_inds[pred_idx++]); + } + } + } else { + for (size_t i = 0; i < conf.dims[1]; i++) { + l = levels[i]; + for (size_t t = 1; t < conf.dims[0]; t++) { + size_t idx = t * conf.dims[1] + i; + l += pred_inds[pred_idx] - level_num; + dec_data[idx] = quantizer.recover(level(l), quant_inds[pred_idx++]); + } + } + } + assert(pred_idx == conf.num); + } + + quantizer.postdecompress_data(); + encoder.postprocess_decode(); + return dec_data; + } + + void set_level(float level_start_, float level_offset_, int level_num_) { + this->level_start = level_start_; + this->level_offset = level_offset_; + this->level_num = level_num_ + 200; + } + + inline int quantize_to_level(T data) { return round((data - level_start) / level_offset); } + + inline T level(int l) { return level_start + nofma(l * level_offset); } + + private: + Quantizer quantizer; + Encoder encoder; + Lossless lossless; + float level_start{}; + float level_offset{}; + int level_num{}; + int timestep_op; +}; + +template +std::shared_ptr> make_compressor_exaalt(Quantizer quantizer, + Encoder encoder, + Lossless lossless, + int timestep_op) { + return std::make_shared>(quantizer, encoder, lossless, + timestep_op); +} +} // namespace SZ3 +#endif diff --git a/include/SZ3/decomposition/BlockwiseDecomposition.hpp b/include/SZ3/decomposition/BlockwiseDecomposition.hpp new file mode 100644 index 00000000..bb7683ea --- /dev/null +++ b/include/SZ3/decomposition/BlockwiseDecomposition.hpp @@ -0,0 +1,106 @@ +#ifndef SZ3_BLOCKWISE_DECOMPOSITION_HPP +#define SZ3_BLOCKWISE_DECOMPOSITION_HPP + +#include +#include +#include +#include + +#include "Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" +#include "SZ3/utils/BlockwiseIterator.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Timer.hpp" + +namespace SZ3 { +template +class BlockwiseDecomposition : public concepts::DecompositionInterface { + public: + using Block_iter = typename block_data::block_iterator; + + BlockwiseDecomposition(const Config &conf, Predictor predictor_, Quantizer quantizer_) + : predictor(predictor_), quantizer(quantizer_), fallback_predictor(conf.absErrorBound) { + static_assert(std::is_base_of, Predictor>::value, + "must implement the Predictor interface"); + // A block the predictor declines falls back to Lorenzo, which reads neighbours the predictor may not pad for. + padding = std::max(predictor.get_padding(), fallback_predictor.get_padding()); + } + + std::vector compress(const Config &conf, T *data) override { + auto data_with_padding = std::make_shared>(data, conf.dims, padding, true); + auto block = data_with_padding->block_iter(conf.blockSize); + std::vector quant_inds; + quant_inds.reserve(conf.num); + do { + concepts::PredictorInterface *predictor_withfallback = &predictor; + if (!predictor.precompress(block)) { + predictor_withfallback = &fallback_predictor; + } + predictor_withfallback->precompress_block_commit(); + Block_iter::foreach (block, [&](T *c, const std::array &index) { + T pred = predictor_withfallback->predict(block, c, index); + quant_inds.push_back(quantizer.quantize_and_overwrite(*c, pred)); + }); + + } while (block.next()); + return quant_inds; + } + + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) override { + if (quant_inds.size() < conf.num) { + throw std::out_of_range("SZ3 blockwise: fewer bins than the grid consumes"); + } + int *quant_inds_pos = &quant_inds[0]; + + auto data_with_padding = std::make_shared>(dec_data, conf.dims, padding, false); + auto block = data_with_padding->block_iter(conf.blockSize); + do { + concepts::PredictorInterface *predictor_withfallback = &predictor; + if (!predictor.predecompress(block)) { + predictor_withfallback = &fallback_predictor; + } + Block_iter::foreach (block, [&](T *c, const std::array &index) { + T pred = predictor_withfallback->predict(block, c, index); + *c = quantizer.recover(pred, *(quant_inds_pos++)); + }); + + } while (block.next()); + + return dec_data; + } + + size_t size_est() override { return fallback_predictor.size_est() + predictor.size_est() + quantizer.size_est(); } + + void save(uchar *&c) override { + fallback_predictor.save(c); + predictor.save(c); + quantizer.save(c); + } + + void load(const uchar *&c, size_t &remaining_length) override { + fallback_predictor.load(c, remaining_length); + predictor.load(c, remaining_length); + quantizer.load(c, remaining_length); + } + + std::pair get_out_range() override { return quantizer.get_out_range(); } + + private: + Predictor predictor; + Quantizer quantizer; + LorenzoPredictor fallback_predictor; + size_t padding; +}; + +template +BlockwiseDecomposition make_decomposition_blockwise(const Config &conf, Predictor predictor, + Quantizer quantizer) { + return BlockwiseDecomposition(conf, predictor, quantizer); +} + +} // namespace SZ3 +#endif diff --git a/include/SZ3/decomposition/Decomposition.hpp b/include/SZ3/decomposition/Decomposition.hpp new file mode 100644 index 00000000..595c84bd --- /dev/null +++ b/include/SZ3/decomposition/Decomposition.hpp @@ -0,0 +1,68 @@ +#ifndef SZ3_DECOMPOSITION_INTERFACE +#define SZ3_DECOMPOSITION_INTERFACE + +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/utils/Config.hpp" + +namespace SZ3::concepts { + +/** + * Decomposition represents the lossy part in compressors (which typically converts original data to integers, e.g., + * prediction + quantization). Note: SZ3 has both Predictor and Decomposition interfaces. You should choose Predictor + * interface only when the new prediction method takes scalar value. In all other cases, use this Decomposition + * interface. + * @tparam Ti original data type + * @tparam To decomposed data type + * @tparam N original data dimension + */ +template > +class DecompositionInterface { + public: + virtual ~DecompositionInterface() = default; + + /** + * predict the data and quantize the error + * @param data original input + * @return quantized prediction error + */ + virtual std::vector compress(const Config &conf, Ti *data) = 0; + + /** + * reverse of compress(), reconstruct the data + * @param quant_inds quantized prediction error + * @param dec_data place to write the reconstructed data + * @return same value with dec_data + */ + virtual Ti *decompress(const Config &conf, std::vector &quant_inds, Ti *dec_data) = 0; + + /** + * serialize the frontend and store it to a buffer + * @param c One large buffer is pre-allocated, and the start location of the serialized frontend in the buffer is + * indicated by c. After saving the frontend to the buffer, this function should change c to indicate the next empty + * location in the buffer + */ + virtual void save(uchar *&c) = 0; + + /** + * deserialize the frontend from a buffer + * @param c start location of the frontend in the buffer + * @param remaining_length the remaining length of the buffer + */ + virtual void load(const uchar *&c, size_t &remaining_length) = 0; + + /// Upper bound on the space save() needs for the current input. + virtual size_t size_est() { return 0; } + + virtual std::pair get_out_range() = 0; + + virtual void print() {} + + // virtual void clear() {}; +}; + +} // namespace SZ3::concepts + +#endif diff --git a/include/SZ3/decomposition/InterpolationDecomposition.hpp b/include/SZ3/decomposition/InterpolationDecomposition.hpp new file mode 100644 index 00000000..ab40a941 --- /dev/null +++ b/include/SZ3/decomposition/InterpolationDecomposition.hpp @@ -0,0 +1,507 @@ +#ifndef SZ3_INTERPOLATION_DECOMPOSITION_HPP +#define SZ3_INTERPOLATION_DECOMPOSITION_HPP + +#include +#include +#include +#include + +#include "Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/BlockwiseIterator.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/FileUtil.hpp" +#include "SZ3/utils/Interpolators.hpp" +#include "SZ3/utils/Iterator.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "SZ3/utils/Timer.hpp" + +namespace SZ3 { +template +class InterpolationDecomposition : public concepts::DecompositionInterface { + static_assert(std::is_floating_point::value, "InterpolationDecomposition only takes a floating-point T"); + + public: + InterpolationDecomposition(const Config & /*conf*/, Quantizer quantizer_) : quantizer(quantizer_) { + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quantizer interface"); + } + + T *decompress(const Config &conf, std::vector &quant_inds_, T *dec_data) override { + // load() read original_dimensions from the payload, and the grid walk below is sized by it while + // dec_data and quant_inds are sized by conf. A tampered value runs off both. + if (conf.dims.size() != N) { + throw std::out_of_range("SZ3 interpolation: configuration dimension count does not match the data"); + } + for (uint i = 0; i < N; i++) { + if (original_dimensions[i] != conf.dims[i]) { + throw std::out_of_range("SZ3 interpolation: stored dimensions do not match the trusted configuration"); + } + } + + init(); + + if (quant_inds_.size() < num_elements) { + throw std::out_of_range("SZ3 interpolation: fewer bins than the grid consumes"); + } + this->quant_inds = quant_inds_.data(); + double eb = quantizer.get_eb(); + + if (anchor_stride == 0) { // check whether used anchor points + *dec_data = quantizer.recover(0, this->quant_inds[quant_index++]); // no anchor points + } else { + recover_anchor_grid(dec_data); // recover anchor points + interp_level--; + } + + for (int level = interp_level; level > 0 && level <= interp_level; level--) { + // set level-wise error bound + if (eb_alpha < 0) { + if (level >= 3) { + quantizer.set_eb(eb * eb_ratio); + } else { + quantizer.set_eb(eb); + } + } else if (eb_alpha >= 1) { + double cur_ratio = pow(eb_alpha, level - 1); + if (cur_ratio > eb_beta) { + cur_ratio = eb_beta; + } + quantizer.set_eb(eb / cur_ratio); + } + size_t stride = 1U << (level - 1); + auto interp_block_size = blocksize * stride; + auto inter_block_range = std::make_shared>( + dec_data, std::begin(original_dimensions), std::end(original_dimensions), interp_block_size, 0); + auto inter_begin = inter_block_range->begin(); + auto inter_end = inter_block_range->end(); + for (auto block = inter_begin; block != inter_end; ++block) { + auto end_idx = block.get_global_index(); + for (uint i = 0; i < N; i++) { + end_idx[i] += interp_block_size; + if (end_idx[i] > original_dimensions[i] - 1) { + end_idx[i] = original_dimensions[i] - 1; + } + } + interpolation( + dec_data, block.get_global_index(), end_idx, interpolators[interp_id], + [&](size_t /*idx*/, T &d, T pred) { d = quantizer.recover(pred, quant_inds_[quant_index++]); }, + direction_sequence_id, stride); + } + } + quantizer.postdecompress_data(); + return dec_data; + } + + // compress given the error bound + std::vector compress(const Config &conf, T *data) override { + std::copy_n(conf.dims.begin(), N, original_dimensions.begin()); + + interp_id = conf.interpAlgo; + direction_sequence_id = conf.interpDirection; + anchor_stride = conf.interpAnchorStride; + blocksize = 32; // a empirical value. Can be very large but not helpful + eb_alpha = conf.interpAlpha; + eb_beta = conf.interpBeta; + + init(); + std::vector quant_inds_vec(num_elements); + quant_inds = quant_inds_vec.data(); + double eb = quantizer.get_eb(); + if (anchor_stride == 0) { // check whether to use anchor points + quant_inds[quant_index++] = quantizer.quantize_and_overwrite(*data, 0); // no + } else { + build_anchor_grid(data); // losslessly saving anchor points + interp_level--; + } + + for (int level = interp_level; level > 0 && level <= interp_level; level--) { + double cur_eb = eb; + // set level-wise error bound + if (eb_alpha < 0) { + if (level >= 3) { + cur_eb = eb * eb_ratio; + } else { + cur_eb = eb; + } + } else if (eb_alpha >= 1) { + double cur_ratio = pow(eb_alpha, level - 1); + if (cur_ratio > eb_beta) { + cur_ratio = eb_beta; + } + cur_eb = eb / cur_ratio; + } + quantizer.set_eb(cur_eb); + size_t stride = 1U << (level - 1); + + auto interp_block_size = blocksize * stride; + + auto inter_block_range = std::make_shared>( + data, std::begin(original_dimensions), std::end(original_dimensions), interp_block_size, 0); + + auto inter_begin = inter_block_range->begin(); + auto inter_end = inter_block_range->end(); + + for (auto block = inter_begin; block != inter_end; ++block) { + auto end_idx = block.get_global_index(); + for (uint i = 0; i < N; i++) { + end_idx[i] += interp_block_size; + if (end_idx[i] > original_dimensions[i] - 1) { + end_idx[i] = original_dimensions[i] - 1; + } + } + + interpolation( + data, block.get_global_index(), end_idx, interpolators[interp_id], + [&](size_t /*idx*/, T &d, T pred) { + quant_inds[quant_index++] = (quantizer.quantize_and_overwrite(d, pred)); + }, + direction_sequence_id, stride); + } + } + quantizer.set_eb(eb); + quantizer.postcompress_data(); + return quant_inds_vec; + } + + size_t size_est() override { + return N * sizeof(uint64_t) + sizeof(blocksize) + sizeof(interp_id) + sizeof(direction_sequence_id) + + quantizer.size_est() + 128; + } + + void save(uchar *&c) override { + for (size_t d : original_dimensions) write(d, c); + write(blocksize, c); + write(interp_id, c); + write(direction_sequence_id, c); + write(anchor_stride, c); + write(eb_alpha, c); + write(eb_beta, c); + + quantizer.save(c); + } + + void load(const uchar *&c, size_t &remaining_length) override { + uint64_t v = 0; + for (size_t &d : original_dimensions) { + read(v, c, remaining_length); + d = v; + } + read(blocksize, c, remaining_length); + read(interp_id, c, remaining_length); + read(direction_sequence_id, c, remaining_length); + read(v, c, remaining_length); + anchor_stride = v; + read(eb_alpha, c, remaining_length); + read(eb_beta, c, remaining_length); + + quantizer.load(c, remaining_length); + } + + std::pair get_out_range() override { return quantizer.get_out_range(); } + + private: + void init() { + quant_index = 0; + // All four come from the compressed payload; interp_id and direction_sequence_id index the fixed + // tables built below. + if (blocksize == 0 || blocksize % 2 != 0) { + throw std::out_of_range("SZ3 interpolation: block size must be a positive even number"); + } + if (interp_id < 0 || static_cast(interp_id) >= interpolators.size()) { + throw std::out_of_range("SZ3 interpolation: interpolator id is out of range"); + } + num_elements = 1; + interp_level = -1; + bool use_anchor = false; + for (uint i = 0; i < N; i++) { + if (interp_level < ceil(log2(original_dimensions[i]))) { + interp_level = static_cast(ceil(log2(original_dimensions[i]))); + } + if (original_dimensions[i] > anchor_stride) + use_anchor = true; + num_elements *= original_dimensions[i]; + } + if (!use_anchor) + anchor_stride = 0; + // log2() and the anchor grid below need a power of two; a stride wider than the data is + // already zero by here. + if (anchor_stride > 0 && (anchor_stride & (anchor_stride - 1)) != 0) { + throw std::out_of_range("SZ3 interpolation: anchor stride must be a power of two"); + } + if (anchor_stride > 0) { + int max_interpolation_level = static_cast(log2(anchor_stride)) + 1; + if (max_interpolation_level <= interp_level) { + interp_level = max_interpolation_level; + } + } + + original_dim_offsets[N - 1] = 1; + for (int i = N - 2; i >= 0; i--) { + original_dim_offsets[i] = original_dim_offsets[i + 1] * original_dimensions[i + 1]; + } + + dim_sequences = std::vector>(); + auto sequence = std::array(); + for (uint i = 0; i < N; i++) { + sequence[i] = i; + } + do { + dim_sequences.push_back(sequence); + } while (std::next_permutation(sequence.begin(), sequence.end())); + if (direction_sequence_id < 0 || static_cast(direction_sequence_id) >= dim_sequences.size()) { + throw std::out_of_range("SZ3 interpolation: direction sequence id is out of range"); + } + } + + void build_anchor_grid(T *data) { // store anchor points. steplength: anchor_stride on each dimension + std::array strides; + std::array begins{0}; + std::fill(strides.begin(), strides.end(), anchor_stride); + foreach + (data, 0, begins, original_dimensions, strides, original_dim_offsets, + [&](T *d) { quant_inds[quant_index++] = quantizer.force_save_unpred(*d); }); + } + + void recover_anchor_grid(T *data) { // recover anchor points. steplength: anchor_stride on each dimension + std::array strides; + std::array begins{0}; + std::fill(strides.begin(), strides.end(), anchor_stride); + foreach + (data, 0, begins, original_dimensions, strides, original_dim_offsets, [&](T *d) { + *d = quantizer.recover_unpred(); + quant_index++; + }); + } + + /** + * Do interpolations along a certain dimension, and move through that dimension only. + * This is the original API, described in the ICDE'21 paper. + */ + template + double interpolation_1d(T *data, size_t begin, size_t end, size_t stride, const std::string &interp_func, + QuantizeFunc &&quantize_func) { + size_t n = (end - begin) / stride + 1; + if (n <= 1) { + return 0; + } + double predict_error = 0; + + size_t stride3x = 3 * stride; + size_t stride5x = 5 * stride; + if (interp_func == "linear" || n < 5) { + // if (pb == PB_predict_overwrite) { + for (size_t i = 1; i + 1 < n; i += 2) { + T *d = data + begin + i * stride; + quantize_func(d - data, *d, interp_linear(*(d - stride), *(d + stride))); + } + if (n % 2 == 0) { + T *d = data + begin + (n - 1) * stride; + if (n < 4) { + quantize_func(d - data, *d, *(d - stride)); + } else { + quantize_func(d - data, *d, interp_linear1(*(d - stride3x), *(d - stride))); + } + } + // } + } else { + T *d; + size_t i; + for (i = 3; i + 3 < n; i += 2) { + d = data + begin + i * stride; + quantize_func(d - data, *d, + interp_cubic(*(d - stride3x), *(d - stride), *(d + stride), *(d + stride3x))); + } + d = data + begin + stride; + quantize_func(d - data, *d, interp_quad_1(*(d - stride), *(d + stride), *(d + stride3x))); + + d = data + begin + i * stride; + quantize_func(d - data, *d, interp_quad_2(*(d - stride3x), *(d - stride), *(d + stride))); + if (n % 2 == 0) { + d = data + begin + (n - 1) * stride; + quantize_func(d - data, *d, interp_quad_3(*(d - stride5x), *(d - stride3x), *(d - stride))); + } + } + + return predict_error; + } + + /** + * Do all interpolations along a certain dimension on the full data grid. Moving on the fastest-dim. + * This is the new API, described in the SIGMOD'24 paper. + */ + template + double interpolation_1d_fastest_dim_first(T *data, const std::array &begin_idx, + const std::array &end_idx, const size_t &direction, + std::array &strides, const size_t &math_stride, + const std::string &interp_func, QuantizeFunc &&quantize_func) { + for (size_t i = 0; i < N; i++) { + if (end_idx[i] < begin_idx[i]) return 0; + } + size_t math_begin_idx = begin_idx[direction], math_end_idx = end_idx[direction]; + size_t n = (math_end_idx - math_begin_idx) / math_stride + 1; + if (n <= 1) { + return 0; + } + double predict_error = 0.0; + size_t offset = 0; + size_t stride = math_stride * original_dim_offsets[direction]; + std::array begins, ends, dim_offsets; + for (size_t i = 0; i < N; i++) { + begins[i] = 0; + ends[i] = end_idx[i] - begin_idx[i] + 1; + dim_offsets[i] = original_dim_offsets[i]; + offset += original_dim_offsets[i] * begin_idx[i]; + } + dim_offsets[direction] = stride; + size_t stride2x = 2 * stride; + if (interp_func == "linear") { + begins[direction] = 1; + ends[direction] = n - 1; + strides[direction] = 2; + foreach + (data, offset, begins, ends, strides, dim_offsets, + [&](T *d) { quantize_func(d - data, *d, interp_linear(*(d - stride), *(d + stride))); }); + if (n % 2 == 0) { + begins[direction] = n - 1; + ends[direction] = n; + foreach + (data, offset, begins, ends, strides, dim_offsets, [&](T *d) { + if (n < 3) + quantize_func(d - data, *d, *(d - stride)); + else + quantize_func(d - data, *d, interp_linear1(*(d - stride2x), *(d - stride))); + }); + } + } else { + size_t stride3x = 3 * stride; + size_t i_start = 3; + begins[direction] = i_start; + ends[direction] = (n >= 3) ? (n - 3) : 0; + strides[direction] = 2; + foreach + (data, offset, begins, ends, strides, dim_offsets, [&](T *d) { + quantize_func(d - data, *d, + interp_cubic(*(d - stride3x), *(d - stride), *(d + stride), *(d + stride3x))); + }); + std::vector boundaries; + boundaries.push_back(1); + if (n % 2 == 1 && n > 3) { + boundaries.push_back(n - 2); + } + if (n % 2 == 0 && n > 4) { + boundaries.push_back(n - 3); + } + if (n % 2 == 0 && n > 2) { + boundaries.push_back(n - 1); + } + for (auto boundary : boundaries) { + begins[direction] = boundary; + ends[direction] = boundary + 1; + foreach + (data, offset, begins, ends, strides, dim_offsets, [&](T *d) { + if (boundary >= 3) { + if (boundary + 3 < n) + quantize_func( + d - data, *d, + interp_cubic(*(d - stride3x), *(d - stride), *(d + stride), *(d + stride3x))); + else if (boundary + 1 < n) + quantize_func(d - data, *d, + interp_quad_2(*(d - stride3x), *(d - stride), *(d + stride))); + else + quantize_func(d - data, *d, interp_linear1(*(d - stride3x), *(d - stride))); + } else { + if (boundary + 3 < n) + quantize_func(d - data, *d, + interp_quad_1(*(d - stride), *(d + stride), *(d + stride3x))); + else if (boundary + 1 < n) + quantize_func(d - data, *d, interp_linear(*(d - stride), *(d + stride))); + else + quantize_func(d - data, *d, *(d - stride)); + } + }); + } + } + return predict_error; + } + + template + double interpolation(T *data, std::array begin, std::array end, + const std::string &interp_func, QuantizeFunc &&quantize_func, const int direction, + size_t stride = 1) { + if constexpr (N == 1) { // old API + return interpolation_1d(data, begin[0], end[0], stride, interp_func, quantize_func); + } else if constexpr (N == 2) { // old API + double predict_error = 0; + size_t stride2x = stride * 2; + const std::array dims = dim_sequences[direction]; + for (size_t j = (begin[dims[1]] ? begin[dims[1]] + stride2x : 0); j <= end[dims[1]]; j += stride2x) { + size_t begin_offset = + begin[dims[0]] * original_dim_offsets[dims[0]] + j * original_dim_offsets[dims[1]]; + predict_error += interpolation_1d( + data, begin_offset, begin_offset + (end[dims[0]] - begin[dims[0]]) * original_dim_offsets[dims[0]], + stride * original_dim_offsets[dims[0]], interp_func, quantize_func); + } + for (size_t i = (begin[dims[0]] ? begin[dims[0]] + stride : 0); i <= end[dims[0]]; i += stride) { + size_t begin_offset = + i * original_dim_offsets[dims[0]] + begin[dims[1]] * original_dim_offsets[dims[1]]; + predict_error += interpolation_1d( + data, begin_offset, begin_offset + (end[dims[1]] - begin[dims[1]]) * original_dim_offsets[dims[1]], + stride * original_dim_offsets[dims[1]], interp_func, quantize_func); + } + return predict_error; + } else if constexpr (N == 3 || N == 4) { // new API (for faster speed) + double predict_error = 0; + size_t stride2x = stride * 2; + const std::array dims = dim_sequences[direction]; + std::array strides; + std::array begin_idx = begin, end_idx = end; + strides[dims[0]] = 1; + for (uint i = 1; i < N; i++) { + begin_idx[dims[i]] = (begin[dims[i]] ? begin[dims[i]] + stride2x : 0); + strides[dims[i]] = stride2x; + } + + predict_error += interpolation_1d_fastest_dim_first(data, begin_idx, end_idx, dims[0], strides, stride, + interp_func, quantize_func); + for (uint i = 1; i < N; i++) { + begin_idx[dims[i]] = begin[dims[i]]; + begin_idx[dims[i - 1]] = (begin[dims[i - 1]] ? begin[dims[i - 1]] + stride : 0); + strides[dims[i - 1]] = stride; + predict_error += interpolation_1d_fastest_dim_first(data, begin_idx, end_idx, dims[i], strides, stride, + interp_func, quantize_func); + } + return predict_error; + } else { + throw std::runtime_error("Unsupported dimension in InterpolationDecomposition"); + } + } + + int interp_level = -1; + int interp_id; + uint blocksize; + std::vector interpolators = {"linear", "cubic"}; + int *quant_inds; + size_t quant_index = 0; + double max_error; + Quantizer quantizer; + size_t num_elements; + std::array original_dimensions; + std::array original_dim_offsets; + std::vector> dim_sequences; + int direction_sequence_id; + size_t anchor_stride = 0; + double eb_alpha = -1; + double eb_beta = -1; + double eb_ratio = 0.5; // To be deprecated +}; + +template +InterpolationDecomposition make_decomposition_interpolation(const Config &conf, Quantizer quantizer) { + return InterpolationDecomposition(conf, quantizer); +} + +} // namespace SZ3 + +#endif diff --git a/include/SZ3/decomposition/NoPredictionDecomposition.hpp b/include/SZ3/decomposition/NoPredictionDecomposition.hpp new file mode 100644 index 00000000..d56042fd --- /dev/null +++ b/include/SZ3/decomposition/NoPredictionDecomposition.hpp @@ -0,0 +1,59 @@ +#ifndef SZ3_NO_PREDICTION_DECOMPOSITION_HPP +#define SZ3_NO_PREDICTION_DECOMPOSITION_HPP + +#include + +#include "Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/Config.hpp" + +namespace SZ3 { +template +class NoPredictionDecomposition : public concepts::DecompositionInterface { + public: + NoPredictionDecomposition(const Config & /*conf*/, Quantizer quantizer_) : quantizer(quantizer_) { + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quantizer interface"); + } + + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) override { + if (quant_inds.size() < conf.num) { + throw std::out_of_range("SZ3 no-prediction: fewer bins than the data has elements"); + } + for (size_t i = 0; i < conf.num; i++) { + dec_data[i] = quantizer.recover(0, quant_inds[i]); + } + quantizer.postdecompress_data(); + return dec_data; + } + + std::vector compress(const Config &conf, T *data) override { + std::vector quant_inds(conf.num); + for (size_t i = 0; i < conf.num; i++) { + quant_inds[i] = quantizer.quantize_and_overwrite(data[i], 0); + } + quantizer.postcompress_data(); + return quant_inds; + } + + size_t size_est() override { return quantizer.size_est() + 64; } + + void save(uchar *&c) override { quantizer.save(c); } + + void load(const uchar *&c, size_t &remaining_length) override { quantizer.load(c, remaining_length); } + + std::pair get_out_range() override { return quantizer.get_out_range(); } + + private: + Quantizer quantizer; +}; + +template +NoPredictionDecomposition make_decomposition_noprediction(const Config &conf, Quantizer quantizer) { + return NoPredictionDecomposition(conf, quantizer); +} + +} // namespace SZ3 + +#endif diff --git a/include/SZ3/decomposition/SZBioMDDecomposition.hpp b/include/SZ3/decomposition/SZBioMDDecomposition.hpp new file mode 100644 index 00000000..07145501 --- /dev/null +++ b/include/SZ3/decomposition/SZBioMDDecomposition.hpp @@ -0,0 +1,796 @@ +#ifndef SZ3_BIOMD_DECOMPOSITION_HPP +#define SZ3_BIOMD_DECOMPOSITION_HPP + +// ALGO_BIOMD: molecular-dynamics coordinates {frames, atoms, 3} (nm, absolute bound) as streams of integer symbols, +// one after the other as [count, symbols]. Every coordinate goes on a lattice, +// q = round(x * inv_step), |x - q step| <= eb, +// and all prediction is integer arithmetic on that lattice, which the decoder repeats exactly. +// * rigid water (O, H1, H2): H1 on the sphere |H1 - O| = r, H2 on the circle that r and the H-H distance leave +// * other atoms: from their bond partner, one of the previous MAX_BOND_OFFSET atoms (intra a delta, else on the sphere +// of the previous frame's bond length), else a delta +// * frame 0 intra; for the frames after it water O's from the previous frame, and per other atom group intra or from +// the previous frame, whichever costs less on a sample of frame 1 +// The layout (which atoms are water, the bond of each other atom) is found on a chunk's first frame and stored with it. +// In a chunk of one frame without water and with under a quarter of its atoms bonded (coarse-grained runs), unbonded +// atoms are their points in their box, like intra water O's. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/decomposition/Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +namespace SZ3 { +namespace biomd { + +// A bond partner is searched among the previous MAX_BOND_OFFSET atoms: topologies list an atom shortly after the atom +// it is bonded to. +constexpr int MAX_BOND_OFFSET = 4; +// Deltas from here on (lattice units) are an escape symbol and a raw byte, which keeps the Huffman alphabet small. +constexpr uint32_t ESCAPE = 2048; +// |q| <= 2^28: displacements stay within 2^29, and every symbol within 32 bits +constexpr double MAX_LATTICE = double(1 << 28); +// The lattice spans |x| < B, the smallest power of two above 2^LATTICE_SPAN_BITS eb (for float 2048 nm at 5e-4 nm, 128 +// nm at 5e-5 nm). A larger span costs ratio, as the step is 2 eb less the float ulp at B (choose_step()); double's ulp +// is small enough for B to reach MAX_LATTICE. +template +constexpr int LATTICE_SPAN_BITS = sizeof(T) == 4 ? 21 : 27; + +// atom groups, each with its own predictor mode for the frames after the first (water O's: the previous frame) +enum { G_WATER_O, G_WATER_H, G_BONDED, G_UNBONDED, NUM_GROUPS }; +// symbol streams: the layout (gaps between water O's, the bond of each other atom); water O from the previous frame; +// water H1 on the sphere around O (face, kept coordinates, radial residual); water H2 on the circle (coordinate and +// side, residuals); bonded atoms intra as deltas from their partner, else on the sphere around it (kept coordinates, +// radial residual); unbonded atoms as deltas +enum { + S_WATER_GAP, + S_BOND_REF, + S_WATER_O, + S_WATER_H1_FACE, + S_WATER_H1_KEPT, + S_WATER_H1_RADIAL, + S_WATER_H2_AXIS, + S_WATER_H2_RESIDUAL, + S_BOND_DELTA, + S_BOND_KEPT, + S_BOND_RADIAL, + S_UNBONDED, + NUM_STREAMS +}; +// the group whose atoms write the stream (-1: the layout streams, one symbol per atom at most), and the most symbols +// one of them (water: one molecule) writes per frame; the two size the streams' buffers +inline constexpr int STREAM_GROUP[NUM_STREAMS] = {-1, -1, G_WATER_O, G_WATER_H, G_WATER_H, G_WATER_H, + G_WATER_H, G_WATER_H, G_BONDED, G_BONDED, G_BONDED, G_UNBONDED}; +inline constexpr int MAX_SYMBOLS_PER_UNIT[NUM_STREAMS] = {1, 1, 3, 1, 2, 1, 1, 2, 3, 2, 1, 3}; + +// Values BIOMD does not code (NaN or Inf outside trailing fill, coordinates beyond the lattice); SZ_compress_bioMD +// stores the chunk losslessly. +struct Fallback : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +inline int64_t round_half_away(double y) { return int64_t(y + std::copysign(0.5, y)); } +inline int64_t isqrt_round(int64_t n) { return n <= 0 ? 0 : int64_t(std::sqrt(double(n)) + 0.5); } +// d = atom i - atom j +inline void displacement(const int32_t *q, size_t i, size_t j, int64_t d[3]) { + for (int c = 0; c < 3; c++) d[c] = int64_t(q[3 * i + c]) - q[3 * j + c]; +} + +// The largest axis of p, dropped by the sphere code, and the other two, kept. +ALWAYS_INLINE void split_axes(const int64_t p[3], int &drop, int &keep0, int &keep1) { + drop = std::llabs(p[1]) > std::llabs(p[0]) ? 1 : 0; + if (std::llabs(p[2]) > std::llabs(p[drop])) drop = 2; + keep0 = (drop + 1) % 3; + keep1 = (drop + 2) % 3; +} + +// ------------------------------------------------------------------------------------------------ layout +enum : uint8_t { K_WATER_O, K_WATER_H, K_OTHER }; +struct Layout { + std::vector kind; // per atom: K_WATER_O (then its H at i + 1, i + 2), K_WATER_H, K_OTHER + std::vector bond; // other atoms: 0 = none, else the offset of the partner + double water_oh = 0, water_hh = 0; // O-H and H-H distances of the water (nm) +}; + +// Rigid water on frame x: kind, water_oh, water_hh. Without water, water_oh stays 0 and detect_layout() clears kind. +template +void detect_water(const T *x, size_t atoms, Layout &layout) { + layout.kind.assign(atoms, K_OTHER); + auto dist2 = [x](size_t a, size_t b) { + float dx = float(x[3 * a] - x[3 * b]), dy = float(x[3 * a + 1] - x[3 * b + 1]), + dz = float(x[3 * a + 2] - x[3 * b + 2]); + return nofma(dx * dx) + nofma(dy * dy) + nofma(dz * dz); + }; + // Candidates: O-H in [0.08, 0.125] nm, H-H in [0.13, 0.2] nm, probed at PROBES places across the frame. Rigid + // water gives sharp O-H / H-H peaks, flexible CH2/NH2 groups broad ones. + constexpr float OH2_MIN = 0.0064f, OH2_MAX = 0.015625f, HH2_MIN = 0.0169f, HH2_MAX = 0.04f; // nm^2 + constexpr size_t PROBES = 300, MIN_COUNT = 16; // fewer candidates, or waters, than MIN_COUNT: no water + std::vector cand_oh, cand_hh; + const size_t probe = std::max(1, atoms / PROBES); + for (size_t i = 0; i + 2 < atoms; i += probe) + // five starts: a probe can land on any site of a 3- to 5-site water, and one of them is then its O + for (size_t o = 0; o < 5 && i + o + 2 < atoms; o++) { + const size_t k = i + o; + const float a = dist2(k, k + 1), b = dist2(k, k + 2), c = dist2(k + 1, k + 2); + if (a > OH2_MIN && a < OH2_MAX && b > OH2_MIN && b < OH2_MAX && c > HH2_MIN && c < HH2_MAX) { + cand_hh.push_back(std::sqrt(c)); + cand_oh.push_back(std::sqrt(a)); + cand_oh.push_back(std::sqrt(b)); + break; + } + } + if (cand_hh.size() < MIN_COUNT) return; + // tolerances (nm), the same at every bound: rigid water keeps its geometry far tighter than `tight`, and `loose` + // stays below the O-H / C-H difference (> 0.01 nm) that separates water from CH2/NH2 + constexpr double tight = 0.001, tol = 0.002, loose = 0.01; + auto peak = [](std::vector v) { // the median of the heaviest window of PEAK_WINDOW + constexpr float PEAK_WINDOW = 0.004f; // nm + std::sort(v.begin(), v.end()); + size_t best = 0, n = 0; + for (size_t lo = 0, hi = 0; hi < v.size(); hi++) { + while (v[hi] - v[lo] > PEAK_WINDOW) lo++; + if (hi - lo + 1 > n) { + n = hi - lo + 1; + best = (lo + hi) / 2; + } + } + return double(v[best]); + }; + const double hh = peak(cand_hh); + std::vector oh_of_hh; + for (size_t c = 0; c < cand_hh.size(); c++) + if (std::fabs(cand_hh[c] - hh) < tol) + oh_of_hh.push_back(cand_oh[2 * c]), oh_of_hh.push_back(cand_oh[2 * c + 1]); + const double oh = peak(oh_of_hh); + // rigidity test: rigid water puts nearly every nearby candidate within `tight` of both peaks, flexible water and + // CH2/NH2 groups under half of them + size_t n_tight = 0, n_loose = 0; + for (size_t c = 0; c < cand_hh.size(); c++) { + const double dh = std::fabs(cand_hh[c] - hh), da = std::fabs(cand_oh[2 * c] - oh), + db = std::fabs(cand_oh[2 * c + 1] - oh); + if (dh < loose && da < loose && db < loose) { + n_loose++; + n_tight += dh < tight && da < tight && db < tight; + } + } + if (n_loose < MIN_COUNT || n_tight * 10 < n_loose * 7) return; + const float oh2_lo = float((oh - tol) * (oh - tol)), oh2_hi = float((oh + tol) * (oh + tol)); + const float hh2_lo = float((hh - tol) * (hh - tol)), hh2_hi = float((hh + tol) * (hh + tol)); + size_t waters = 0; + for (size_t i = 0; i + 2 < atoms;) { + float a = dist2(i, i + 1), b, c; + if (a > oh2_lo && a < oh2_hi && (b = dist2(i, i + 2)) > oh2_lo && b < oh2_hi && + (c = dist2(i + 1, i + 2)) > hh2_lo && c < hh2_hi) { + layout.kind[i] = K_WATER_O; + layout.kind[i + 1] = layout.kind[i + 2] = K_WATER_H; + waters++; + i += 3; + } else { + i++; + } + } + if (waters < MIN_COUNT) return; + layout.water_oh = oh; + layout.water_hh = hh; +} + +// Bonds of the other atoms on frame x: each takes the nearest of its previous MAX_BOND_OFFSET atoms if that is within +// 0.01 .. 0.2 nm (covalent bonds, and the virtual site of 4-site water at 0.015 nm from its O; coarse-grained bonds are +// longer). Under one bonded atom in 16 among other atoms probed across the frame, the S_BOND_REF symbols (one per +// atom) would cost more than the bonds save, and none is kept. +template +void detect_bonds(const T *x, size_t atoms, Layout &layout) { + auto partner = [&](size_t i) { // the offset of the bond partner of other atom i, or 0 + float best = 1e30f; + uint8_t offset = 0; // of the nearest previous atom + for (unsigned o = 1; o <= unsigned(MAX_BOND_OFFSET) && o <= i; o++) { + const float dx = float(x[3 * i] - x[3 * (i - o)]), dy = float(x[3 * i + 1] - x[3 * (i - o) + 1]), + dz = float(x[3 * i + 2] - x[3 * (i - o) + 2]), + v = nofma(dx * dx) + nofma(dy * dy) + nofma(dz * dz); + offset = v < best ? uint8_t(o) : offset; + best = std::min(best, v); + } + return best > 0.0001f && best < 0.04f ? offset : uint8_t(0); + }; + layout.bond.assign(atoms, 0); + size_t probed = 0, bonded = 0; + for (size_t i = 1; i < atoms; i += std::max(1, atoms / 256)) + if (layout.kind[i] == K_OTHER) probed++, bonded += partner(i) != 0; + if (bonded * 16 < probed) return; + for (size_t i = 1; i < atoms; i++) + if (layout.kind[i] == K_OTHER) layout.bond[i] = partner(i); +} + +// ------------------------------------------------------------------------------------------------ symbols +// the position in each stream, and raw bits (LSB first) +struct SymbolWriter { + int *cursor[NUM_STREAMS]; + BitAppender *raw; + ALWAYS_INLINE void put(int s, uint32_t v) { *cursor[s]++ = int(v); } + ALWAYS_INLINE void put_bits(uint64_t v, int bits) { raw->put(v, bits); } // bits <= 56 +}; +// bits of a sample, for the predictor choice +struct CostEstimate { + double bits = 0; + inline void put(int, uint32_t v) { bits += std::log2(double(v) + 1.0) + 1; } + inline void put_bits(uint64_t, int n) { bits += n; } +}; +// a cursor per stream, and one on the raw bits +struct SymbolReader { + const int *cursor[NUM_STREAMS], *end[NUM_STREAMS]; + BitConsumer raw{nullptr, nullptr}; + ALWAYS_INLINE uint32_t get(int s) { + if (cursor[s] == end[s]) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + return uint32_t(*cursor[s]++); + } + ALWAYS_INLINE uint64_t get_bits(int bits) { return raw.get(bits); } // bits <= 56 +}; + +// v as one symbol below ESCAPE, else as ESCAPE + v / 256 and its low byte as raw bits +template +ALWAYS_INLINE void put_escaped(Sink &out, int s, uint32_t v) { + if (v < ESCAPE) return out.put(s, v); + out.put(s, ESCAPE + (v >> 8)); + out.put_bits(v & 255, 8); +} +ALWAYS_INLINE uint32_t get_escaped(SymbolReader &in, int s) { + const uint32_t v = in.get(s); + return v < ESCAPE ? v : (v - ESCAPE) << 8 | uint32_t(in.get_bits(8)); +} + +struct FrameContext { + int32_t *cur, *prev; // this frame and the one before, on the lattice + const Layout *layout; + int64_t water_oh2, water_hh2; // squared O-H and H-H distances of the water (lattice units) + // the box of the water O's: corner, sides, and the bits of a point in it (x + Rx (y + Ry z)), or -1 if that + // takes more than 56: then each coordinate in the bits of its side + int32_t box_min[3]; + uint64_t box_size[3]; + int box_bits, side_bits[3]; + bool unbonded_box; // unbonded atoms as their points in the box +}; +inline void set_water_box(FrameContext &frame, const uint32_t *size) { // sides of at most 2^30 + uint64_t *R = frame.box_size; + for (int c = 0; c < 3; c++) R[c] = size[c]; + // bits of the largest value below n + auto bits_below = [](uint64_t n) { return int(vector_bit_width(std::vector{n - 1})); }; + frame.box_bits = R[0] * R[1] < (uint64_t(1) << 56) / R[2] ? bits_below(R[0] * R[1] * R[2]) : -1; + for (int c = 0; c < 3; c++) frame.side_bits[c] = bits_below(R[c]); +} + +// a water O of an intra frame: its point in the box +template +inline void put_box_point(const FrameContext &frame, const int32_t *p, Sink &out) { + const uint64_t *R = frame.box_size; + uint64_t u[3]; + for (int c = 0; c < 3; c++) u[c] = uint64_t(int64_t(p[c]) - frame.box_min[c]); + if (frame.box_bits >= 0) return out.put_bits(u[0] + R[0] * (u[1] + R[1] * u[2]), frame.box_bits); + for (int c = 0; c < 3; c++) out.put_bits(u[c], frame.side_bits[c]); +} +inline void get_box_point(const FrameContext &frame, int32_t *p, SymbolReader &in) { + const uint64_t *R = frame.box_size; + uint64_t u[3]; + if (frame.box_bits >= 0) { + const uint64_t v = in.get_bits(frame.box_bits), yz = v / R[0]; + u[0] = v - yz * R[0]; + u[1] = yz % R[1]; + u[2] = yz / R[1]; + } else { + for (int c = 0; c < 3; c++) u[c] = in.get_bits(frame.side_bits[c]); + } + for (int c = 0; c < 3; c++) p[c] = int32_t(frame.box_min[c] + int64_t(u[c])); +} + +// coordinate c of unbonded atom i: from the atom before (intra) or from the previous frame +ALWAYS_INLINE int64_t predict_unbonded(const FrameContext &frame, int mode, size_t i, int c) { + return mode ? frame.prev[3 * i + c] : (i ? frame.cur[3 * (i - 1) + c] : 0); +} + +// Atom i on the sphere |d|^2 = r2 around atom j (r2 = 0: |p|^2 up to 2^28, mode 1 alone): the largest axis of d is +// dropped and rebuilt from the other two, with a radial residual. Intra (mode 0) the face (2 axis + sign) is sent with +// the residual and the two kept coordinates as they are; else the vector p of the previous frame fixes the axis and the +// sign, and the kept coordinates are residuals against p. +template +ALWAYS_INLINE void put_sphere(const FrameContext &frame, int mode, size_t i, size_t j, int64_t r2, int s_kept, + int s_radial, Sink &out) { + int64_t d[3], p[3] = {0, 0, 0}; + int drop, k0, k1; + displacement(frame.cur, i, j, d); + if (mode) displacement(frame.prev, i, j, p); + if (!r2) r2 = std::min(p[0] * p[0] + p[1] * p[1] + p[2] * p[2], int64_t(1) << 28); + split_axes(mode ? p : d, drop, k0, k1); + const bool neg = (mode ? p[drop] : d[drop]) < 0; + const uint32_t radial = zigzag((neg ? -d[drop] : d[drop]) - isqrt_round(r2 - d[k0] * d[k0] - d[k1] * d[k1])); + if (mode == 0) out.put(S_WATER_H1_FACE, uint32_t(2 * drop + neg) + 6 * std::min(radial, 15u)); + if (mode != 0 || radial >= 15) out.put(s_radial, mode == 0 ? radial - 15 : radial); + out.put(s_kept, zigzag(d[k0] - p[k0])); + out.put(s_kept, zigzag(d[k1] - p[k1])); +} +ALWAYS_INLINE void get_sphere(const FrameContext &frame, int mode, size_t i, size_t j, int64_t r2, int s_kept, + int s_radial, SymbolReader &in) { + int64_t d[3], p[3] = {0, 0, 0}; + int drop, k0, k1; + bool neg; + uint32_t radial; + if (mode == 0) { + const uint32_t v = in.get(S_WATER_H1_FACE); + drop = int(v % 6 / 2); + k0 = (drop + 1) % 3; + k1 = (drop + 2) % 3; + neg = v & 1; + radial = v / 6; + if (radial == 15) radial += in.get(s_radial); + } else { + displacement(frame.prev, i, j, p); + if (!r2) r2 = std::min(p[0] * p[0] + p[1] * p[1] + p[2] * p[2], int64_t(1) << 28); + split_axes(p, drop, k0, k1); + neg = p[drop] < 0; + radial = in.get(s_radial); + } + d[k0] = p[k0] + unzigzag(in.get(s_kept)); + d[k1] = p[k1] + unzigzag(in.get(s_kept)); + // unsigned: a corrupt stream's squares may pass 2^63, a valid one's stay below 2^60 + const uint64_t rest = uint64_t(r2) - uint64_t(d[k0]) * uint64_t(d[k0]) - uint64_t(d[k1]) * uint64_t(d[k1]); + const int64_t r = isqrt_round(int64_t(rest)) + unzigzag(radial); + d[drop] = neg ? -r : r; + for (int c = 0; c < 3; c++) frame.cur[3 * i + c] = int32_t(frame.cur[3 * j + c] + d[c]); +} + +// H2 - O = d on the circle |d|^2 = oh2, 2 d.h1 = oh2 + |h1|^2 - hh2, h1 = H1 - O. The coordinate of d on the axis +// where h1 is smallest leaves two points of the circle, one on each side of the plane through h1 and that axis; the +// side of d (intra: sent, else as a change from the side of the prediction p) picks one, and two residuals correct it. +// Integer products and a square root and a division of exact doubles, so the decoder finds the same point. Beyond the +// lattice sizes of a water (a molecule split by the boundary) the point is {coord on the axis, 0, 0}. +ALWAYS_INLINE int circle_axis(const int64_t h1[3]) { + int axis = std::llabs(h1[1]) < std::llabs(h1[0]) ? 1 : 0; + return std::llabs(h1[2]) < std::llabs(h1[axis]) ? 2 : axis; +} +// the side of v: 1 when (-h1[j], h1[i]) . (v[i], v[j]) < 0, for the other two axes i, j +ALWAYS_INLINE int circle_side(const int64_t h1[3], int axis, const int64_t v[3]) { + const int i = (axis + 1) % 3, j = (axis + 2) % 3; + return h1[i] * v[j] - h1[j] * v[i] < 0; +} +ALWAYS_INLINE void circle_point(const int64_t h1[3], int64_t oh2, int64_t hh2, int axis, int64_t coord, int side, + int64_t point[3]) { + const int i = (axis + 1) % 3, j = (axis + 2) % 3; + const int64_t u = h1[i], w = h1[j], lim = 16384; + const bool ok = std::llabs(u) < lim && std::llabs(w) < lim && std::llabs(h1[axis]) < lim && + std::llabs(coord) < lim && u * u + w * w > 0; + const int64_t g2 = ok ? u * u + w * w : 1; + const int64_t M = ok ? oh2 + g2 + h1[axis] * h1[axis] - hh2 - 2 * h1[axis] * coord : 0; // 2 (u P_i + w P_j) + const int64_t s = ok ? (side ? -1 : 1) * isqrt_round(4 * g2 * (oh2 - coord * coord) - M * M) : 0; + point[axis] = coord; + point[i] = ok ? round_half_away(double(M * u - s * w) / double(2 * g2)) : 0; + point[j] = ok ? round_half_away(double(M * w + s * u) / double(2 * g2)) : 0; +} +// H2 - O of the previous frame, or 0 beyond the lattice sizes of a water, so that 2 zigzag(d - p) + 1 fits 32 bits +ALWAYS_INLINE void previous_h2(const FrameContext &frame, size_t o, int64_t p[3]) { + displacement(frame.prev, o + 2, o, p); + if (std::llabs(p[0]) >= 16384 || std::llabs(p[1]) >= 16384 || std::llabs(p[2]) >= 16384) p[0] = p[1] = p[2] = 0; +} +template +ALWAYS_INLINE void put_circle(const FrameContext &frame, int mode, size_t o, Sink &out) { + int64_t h1[3], d[3], p[3], point[3]; + displacement(frame.cur, o + 1, o, h1); + displacement(frame.cur, o + 2, o, d); + const int axis = circle_axis(h1), side = circle_side(h1, axis, d); + if (mode) { + previous_h2(frame, o, p); + out.put(S_WATER_H2_AXIS, zigzag(d[axis] - p[axis]) * 2 + uint32_t(side != circle_side(h1, axis, p))); + } else { + out.put(S_WATER_H2_AXIS, zigzag(d[axis]) * 2 + uint32_t(side)); + } + circle_point(h1, frame.water_oh2, frame.water_hh2, axis, d[axis], side, point); + for (int c = 0; c < 3; c++) + if (c != axis) out.put(S_WATER_H2_RESIDUAL, zigzag(d[c] - point[c])); +} +ALWAYS_INLINE void get_circle(const FrameContext &frame, int mode, size_t o, SymbolReader &in) { + int64_t h1[3], p[3] = {0, 0, 0}, point[3]; + displacement(frame.cur, o + 1, o, h1); + const int axis = circle_axis(h1); + if (mode) previous_h2(frame, o, p); + const uint32_t v = in.get(S_WATER_H2_AXIS); + const int side = int(v & 1) ^ (mode ? circle_side(h1, axis, p) : 0); + circle_point(h1, frame.water_oh2, frame.water_hh2, axis, p[axis] + unzigzag(v >> 1), side, point); + for (int c = 0; c < 3; c++) + frame.cur[3 * (o + 2) + c] = + int32_t(frame.cur[3 * o + c] + (c == axis ? point[c] : point[c] + unzigzag(in.get(S_WATER_H2_RESIDUAL)))); +} + +// A water O: from the previous frame, or (intra) its point in the box. +template +ALWAYS_INLINE void put_water_o(const FrameContext &frame, int mode, size_t i, Sink &out) { + if (mode) + for (int c = 0; c < 3; c++) out.put(S_WATER_O, zigzag(frame.cur[3 * i + c] - frame.prev[3 * i + c])); + else + put_box_point(frame, frame.cur + 3 * i, out); +} + +// One atom: a water (O and the rest of its molecule), or another atom. Water H come with their O. +template +ALWAYS_INLINE void put_atom(const FrameContext &frame, const int *mode, size_t i, Sink &out) { + const Layout &layout = *frame.layout; + const int32_t *q = frame.cur; + if (layout.kind[i] == K_WATER_O) { + put_water_o(frame, mode[G_WATER_O], i, out); + put_sphere(frame, mode[G_WATER_H], i + 1, i, frame.water_oh2, S_WATER_H1_KEPT, S_WATER_H1_RADIAL, out); + put_circle(frame, mode[G_WATER_H], i, out); + } else if (layout.kind[i] == K_OTHER) { + const size_t j = i - layout.bond[i]; + if (j != i && mode[G_BONDED]) + put_sphere(frame, 1, i, j, 0, S_BOND_KEPT, S_BOND_RADIAL, out); + else if (j != i) + for (int c = 0; c < 3; c++) put_escaped(out, S_BOND_DELTA, zigzag(q[3 * i + c] - q[3 * j + c])); + else if (frame.unbonded_box) + put_box_point(frame, q + 3 * i, out); + else + for (int c = 0; c < 3; c++) + put_escaped(out, S_UNBONDED, zigzag(q[3 * i + c] - predict_unbonded(frame, mode[G_UNBONDED], i, c))); + } +} +ALWAYS_INLINE void get_atom(const FrameContext &frame, const int *mode, size_t i, SymbolReader &in) { + const Layout &layout = *frame.layout; + int32_t *q = frame.cur; + if (layout.kind[i] == K_WATER_O) { + if (mode[G_WATER_O]) + for (int c = 0; c < 3; c++) q[3 * i + c] = int32_t(frame.prev[3 * i + c] + unzigzag(in.get(S_WATER_O))); + else + get_box_point(frame, q + 3 * i, in); + get_sphere(frame, mode[G_WATER_H], i + 1, i, frame.water_oh2, S_WATER_H1_KEPT, S_WATER_H1_RADIAL, in); + get_circle(frame, mode[G_WATER_H], i, in); + } else if (layout.kind[i] == K_OTHER) { + const size_t j = i - layout.bond[i]; + if (j != i && mode[G_BONDED]) + get_sphere(frame, 1, i, j, 0, S_BOND_KEPT, S_BOND_RADIAL, in); + else if (j != i) + for (int c = 0; c < 3; c++) q[3 * i + c] = int32_t(q[3 * j + c] + unzigzag(get_escaped(in, S_BOND_DELTA))); + else if (frame.unbonded_box) + get_box_point(frame, q + 3 * i, in); + else + for (int c = 0; c < 3; c++) + q[3 * i + c] = + int32_t(predict_unbonded(frame, mode[G_UNBONDED], i, c) + unzigzag(get_escaped(in, S_UNBONDED))); + } +} + +// q = round(x * inv_step), inv_step the reciprocal of the step +template +inline void quantize(const T *x, size_t n, double inv_step, int32_t *q) { + for (size_t i = 0; i < n; i++) { + const double y = double(x[i]) * inv_step; + q[i] = int32_t(y + std::copysign(0.5, y)); + } +} + +// the box of the water O's of frame q +inline void water_box(const int32_t *q, const std::vector &waters, int32_t lo[3], int32_t hi[3]) { + for (int c = 0; c < 3; c++) lo[c] = hi[c] = waters.empty() ? 0 : q[3 * waters[0] + c]; + for (uint32_t i : waters) + for (int c = 0; c < 3; c++) { + lo[c] = std::min(lo[c], q[3 * i + c]); + hi[c] = std::max(hi[c], q[3 * i + c]); + } +} + +// Per group but the water O's, the mode (0 intra, 1 previous frame) that costs fewer bits on a sample of about 256 of +// its atoms. +inline void choose_modes(const FrameContext &frame, const std::vector *const group_atoms[NUM_GROUPS], + int mode[NUM_GROUPS]) { + for (int g = G_WATER_H; g < NUM_GROUPS; g++) { + const auto &sample = *group_atoms[g]; + double best = 1e300; + for (int m = 0; m < 2 && !sample.empty(); m++) { + CostEstimate cost; + int trial[NUM_GROUPS] = {mode[0], mode[1], mode[2], mode[3]}; + trial[g] = m; + const size_t stride = std::max(1, sample.size() / 256); + for (size_t k = 0; k < sample.size(); k += stride) put_atom(frame, trial, sample[k], cost); + if (cost.bits < best) best = cost.bits, mode[g] = m; + } + } +} + +} // namespace biomd + +template +class SZBioMDDecomposition : public concepts::DecompositionInterface { + static_assert(std::is_floating_point::value, "SZ3 BioMD: data must be float or double"); + + public: + explicit SZBioMDDecomposition(const Config &conf) + : frames_(N == 3 ? conf.dims[0] : 1), atoms_(N >= 2 ? conf.dims[N - 2] : 1), error_bound_(conf.absErrorBound) { + if (N > 3 || conf.dims[N - 1] != 3) throw std::invalid_argument("SZ3 BioMD: data must be {frames, atoms, 3}"); + } + + // The symbol streams of the chunk, one after the other as [count, symbols]; the rest goes to save(). + std::vector compress(const Config & /*conf*/, T *data) override { + using namespace biomd; + const size_t frame_values = atoms_ * 3; + find_fill_frames(data); + choose_step(data); + Layout layout; + std::vector waters, bonded, unbonded; // water O's, other atoms + detect_layout(data, layout, waters, bonded, unbonded); + // one frame without water and mostly unbonded (coarse-grained): box points cost about the bits of deltas, and + // less time + unbonded_box_ = coded_frames_ == 1 && waters.empty() && bonded.size() * 4 < atoms_; + + // a buffer per stream, as large as the symbols the chunk can put in it (every value is written before it is + // read) + const std::vector *group_atoms[NUM_GROUPS] = {&waters, &waters, &bonded, &unbonded}; + std::unique_ptr buffers[NUM_STREAMS]; + SymbolWriter writer; + for (int s = 0; s < NUM_STREAMS; s++) { + const size_t n = STREAM_GROUP[s] < 0 ? atoms_ : coded_frames_ * group_atoms[STREAM_GROUP[s]]->size(); + buffers[s].reset(new int[MAX_SYMBOLS_PER_UNIT[s] * n + 1]); + writer.cursor[s] = buffers[s].get(); + } + raw_bits_.clear(); + BitAppender raw(raw_bits_); + writer.raw = &raw; + for (size_t k = 0; k < waters.size(); k++) writer.put(S_WATER_GAP, waters[k] - (k ? waters[k - 1] : 0)); + for (size_t i = 0; i < atoms_; i++) + if (layout.kind[i] == K_OTHER) writer.put(S_BOND_REF, layout.bond[i]); + + std::unique_ptr lattice(new int32_t[2 * frame_values]); // written before read + FrameContext frame = frame_context(lattice.get(), layout); + const int intra[NUM_GROUPS] = {0, 0, 0, 0}; + std::fill(modes_ + G_WATER_H, modes_ + NUM_GROUPS, 0); + for (size_t t = 0; t < coded_frames_; t++) { + quantize(data + t * frame_values, frame_values, 1.0 / step_, frame.cur); + if (t == 0) { // sides of 1 without water O's or unbonded box points + int32_t box_max[3]; + water_box(frame.cur, unbonded_box_ ? unbonded : waters, frame.box_min, box_max); + for (int c = 0; c < 3; c++) box_min_[c] = frame.box_min[c], box_size_[c] = box_max[c] - box_min_[c] + 1; + set_water_box(frame, box_size_); + } + // per group, the predictor that is cheapest on a sample of frame 1, for every frame after the first + if (t == 1) choose_modes(frame, group_atoms, modes_); + for (size_t i = 0; i < atoms_; i++) put_atom(frame, t ? modes_ : intra, i, writer); + std::swap(frame.cur, frame.prev); + } + raw.flush(); + + std::vector bins; + size_t n = NUM_STREAMS; + for (int s = 0; s < NUM_STREAMS; s++) { + if (size_t(writer.cursor[s] - buffers[s].get()) > size_t(std::numeric_limits::max())) + throw std::invalid_argument("SZ3 BioMD: a stream of more symbols than an int counts"); + n += size_t(writer.cursor[s] - buffers[s].get()); + } + bins.reserve(n); + for (int s = 0; s < NUM_STREAMS; s++) { + bins.push_back(int(writer.cursor[s] - buffers[s].get())); + bins.insert(bins.end(), buffers[s].get(), writer.cursor[s]); + } + return bins; + } + + // The chunk from its symbol streams, each as [count, symbols], after load(). + T *decompress(const Config & /*conf*/, std::vector &quant_inds, T *dec_data) override { + using namespace biomd; + const size_t frame_values = atoms_ * 3; + SymbolReader reader; + for (size_t k = 0, s = 0; s < NUM_STREAMS; s++) { + const size_t n = k < quant_inds.size() ? size_t(quant_inds[k]) : 0; + if (k >= quant_inds.size() || n > quant_inds.size() - k - 1) + throw std::runtime_error("SZ3 BioMD: corrupt stream"); + reader.cursor[s] = quant_inds.data() + k + 1; + reader.end[s] = reader.cursor[s] + n; + k += n + 1; + } + reader.raw = BitConsumer(raw_bits_.data(), raw_bits_.data() + raw_bits_.size()); + Layout layout; + read_layout(reader, layout); + std::unique_ptr lattice(new int32_t[2 * frame_values]); // written before read + FrameContext frame = frame_context(lattice.get(), layout); + const int intra[NUM_GROUPS] = {0, 0, 0, 0}; + std::copy(box_min_, box_min_ + 3, frame.box_min); + set_water_box(frame, box_size_); + for (size_t t = 0; t < coded_frames_; t++) { + for (size_t i = 0; i < atoms_; i += layout.kind[i] == K_WATER_O ? 3 : 1) // a water's H come with its O + get_atom(frame, t ? modes_ : intra, i, reader); + T *x = dec_data + t * frame_values; + for (size_t i = 0; i < frame_values; i++) x[i] = T(double(frame.cur[i]) * step_); + std::swap(frame.cur, frame.prev); + } + // a stream that holds more than the frames took is corrupt + for (int s = 0; s < NUM_STREAMS; s++) + if (reader.cursor[s] != reader.end[s]) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + if (!reader.raw.at_end()) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + std::fill(dec_data + coded_frames_ * frame_values, dec_data + frames_ * frame_values, fill_value_); + return dec_data; + } + + void save(uchar *&c) override { + write(uint64_t(coded_frames_), c); + write(fill_value_, c); + write(step_, c); + write(water_oh2_, c); + write(water_hh2_, c); + write(uint64_t(num_waters_), c); + for (int m : modes_) write(uint8_t(m), c); + write(uint8_t(unbonded_box_), c); + if (has_box()) { + write(box_min_, 3, c); + write(box_size_, 3, c); + } + write(uint64_t(raw_bits_.size()), c); + if (!raw_bits_.empty()) write(raw_bits_.data(), raw_bits_.size(), c); + } + + void load(const uchar *&c, size_t &remaining_length) override { + uint64_t waters, coded, raw_size = 0; + read(coded, c, remaining_length); + if (coded > frames_) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + coded_frames_ = size_t(coded); + read(fill_value_, c, remaining_length); + read(step_, c, remaining_length); + read(water_oh2_, c, remaining_length); + read(water_hh2_, c, remaining_length); + read(waters, c, remaining_length); + num_waters_ = size_t(std::min(waters, atoms_)); + for (int &m : modes_) { + uint8_t v; + read(v, c, remaining_length); + if (v > 1) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + m = v; + } + uint8_t unbonded_box; + read(unbonded_box, c, remaining_length); + if (unbonded_box > 1) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + unbonded_box_ = unbonded_box && coded_frames_ == 1; + if (has_box()) { + read(box_min_, 3, c, remaining_length); + read(box_size_, 3, c, remaining_length); + } + read(raw_size, c, remaining_length); + if (raw_size > remaining_length) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + raw_bits_.resize(size_t(raw_size)); + if (raw_size) read(raw_bits_.data(), raw_bits_.size(), c, remaining_length); + bool ok = std::isfinite(step_) && step_ > 0 && waters <= atoms_ && water_oh2_ >= 0 && + water_oh2_ < (int64_t(1) << 28) && water_hh2_ >= 0 && water_hh2_ < (int64_t(1) << 30); + for (uint32_t side : box_size_) ok = ok && side >= 1 && side <= (uint32_t(1) << 30); + if (!ok) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + } + + // a bound on what save() writes + size_t size_est() override { return 96 + raw_bits_.size(); } + + // every bin, a count or a symbol, is in [0, INT_MAX] + std::pair get_out_range() override { return {0, std::numeric_limits::max()}; } + + private: + // frame 0 stores the box of its water O's, or of its unbonded atoms + bool has_box() const { return unbonded_box_ || num_waters_; } + + // Trailing frames all of one value (the unwritten rest of a chunk) are stored as that value; a chunk may be all + // fill (an unwritten chunk of NaN). + void find_fill_frames(const T *data) { + const size_t frame_values = atoms_ * 3; + fill_value_ = data[(frames_ - 1) * frame_values]; + auto is_fill = [&](size_t t) { + for (size_t i = t * frame_values; i < (t + 1) * frame_values; i++) + if (memcmp(&data[i], &fill_value_, sizeof(T)) != 0) return false; + return true; + }; + for (coded_frames_ = frames_; coded_frames_ > 0 && is_fill(coded_frames_ - 1);) coded_frames_--; + } + + // Lattice step: 2 eb - m ulp for the binade below B, so that q step rounded to T stays within eb for every + // |q step| < B. Rounding q step to float adds ulp / 2 to the step / 2 of the lattice, and the double arithmetic + // (x * inv_step rounded to q is off by up to 2 double ulps of x, q step by 1 more) under 2^-20 float ulps: m = 1 + + // 2^-10 for float. For double those ulps are its own: m = 8. B, the power of two above + // 2^LATTICE_SPAN_BITS eb, depends on the bound alone, so a decompressed chunk compressed again lands on the same + // lattice points. A chunk with NaN or Inf goes to lossless compression; one with coordinates beyond B, or under a + // bound for which m ulp reaches eb or B passes the largest value of T, also to lossless compression. + void choose_step(const T *data) { + if (!(error_bound_ > 0) || !std::isfinite(error_bound_)) + throw std::invalid_argument("SZ3 BioMD: the error bound must be positive and finite"); + using Bits = typename std::conditional::type; + Bits max_bits = 0; // the largest |x| as bits: NaN and inf are above every finite value + for (size_t i = 0; i < coded_frames_ * atoms_ * 3; i++) { + Bits b; + memcpy(&b, &data[i], sizeof(T)); + b &= Bits(~Bits(0)) >> 1; + max_bits = b > max_bits ? b : max_bits; + } + // compared as bits: under -ffinite-math-only a floating-point test for NaN may be compiled away + const Bits infinity_bits = Bits(sizeof(T) == 4 ? 0x7f800000ull : 0x7ff0000000000000ull); + if (max_bits >= infinity_bits) + throw biomd::Fallback("SZ3 BioMD: NaN or Inf in a frame that is not trailing fill"); + T max_abs; + memcpy(&max_abs, &max_bits, sizeof(T)); + int exponent; + std::frexp(std::ldexp(error_bound_, biomd::LATTICE_SPAN_BITS), &exponent); + const double span = std::ldexp(1.0, exponent); + const double margin = (sizeof(T) == 4 ? 1 + 1.0 / 1024 : 8) * + std::max(double(std::numeric_limits::denorm_min()), + std::ldexp(1.0, exponent - std::numeric_limits::digits)); + step_ = 2.0 * error_bound_ - margin; + // every lattice point |q| step below B, with q rounded as quantize() does (after max|x| < B + step, which + // keeps q within MAX_LATTICE): decompressed values land on the same points, and compress the same again + if (!(margin < error_bound_) || !std::isfinite(1.0 / step_) || + !(span <= double(std::numeric_limits::max())) || !(span / step_ < biomd::MAX_LATTICE) || + !(double(max_abs) < span + step_) || + !(nofma(std::fabs(double(biomd::round_half_away(nofma(double(max_abs) * (1.0 / step_))))) * step_) < span)) + throw biomd::Fallback("SZ3 BioMD: coordinates beyond the lattice the error bound allows"); + } + + // The layout of the first frame, its geometry on the lattice, and the atoms of each group. + void detect_layout(const T *data, biomd::Layout &layout, std::vector &waters, + std::vector &bonded, std::vector &unbonded) { + using namespace biomd; + detect_water(data, atoms_, layout); + const double oh = layout.water_oh / step_, hh = layout.water_hh / step_; + water_oh2_ = oh < 16384 ? round_half_away(nofma(oh * oh)) : 0; // squares that load() accepts + water_hh2_ = hh < 32768 ? round_half_away(nofma(hh * hh)) : 0; + if (!(water_oh2_ > 0 && water_oh2_ < (int64_t(1) << 28) && water_hh2_ > 0 && water_hh2_ < (int64_t(1) << 30))) { + water_oh2_ = water_hh2_ = 0; // no water, or too many lattice steps across one for exact products + std::fill(layout.kind.begin(), layout.kind.end(), K_OTHER); + } + detect_bonds(data, atoms_, layout); + for (size_t i = 0; i < atoms_; i++) { + if (layout.kind[i] == K_WATER_O) waters.push_back(uint32_t(i)); + if (layout.kind[i] == K_OTHER) (layout.bond[i] ? bonded : unbonded).push_back(uint32_t(i)); + } + num_waters_ = waters.size(); + } + + // The layout from its streams: the water O's by their gaps, then the bond of each other atom. + void read_layout(biomd::SymbolReader &reader, biomd::Layout &layout) const { + using namespace biomd; + layout.kind.assign(atoms_, K_OTHER); + for (size_t k = 0, o = 0; k < num_waters_; k++) { + const uint32_t gap = reader.get(S_WATER_GAP); + if ((k && gap < 3) || gap > atoms_ || o + gap + 2 >= atoms_) + throw std::runtime_error("SZ3 BioMD: corrupt stream"); + o += gap; + layout.kind[o] = K_WATER_O; + layout.kind[o + 1] = layout.kind[o + 2] = K_WATER_H; + } + layout.bond.assign(atoms_, 0); + for (size_t i = 0; i < atoms_; i++) + if (layout.kind[i] == K_OTHER) { + const uint32_t bond = reader.get(S_BOND_REF); + if (bond > uint32_t(MAX_BOND_OFFSET) || bond > i) throw std::runtime_error("SZ3 BioMD: corrupt stream"); + layout.bond[i] = uint8_t(bond); + } + } + + // the lattice holds this frame and the one before + biomd::FrameContext frame_context(int32_t *lattice, const biomd::Layout &layout) const { + return {lattice, lattice + atoms_ * 3, &layout, water_oh2_, water_hh2_, {0, 0, 0}, {1, 1, 1}, 0, + {0, 0, 0}, unbonded_box_}; + } + + size_t frames_, atoms_, coded_frames_ = 1; // coded: the frames before the fill + T fill_value_ = 0; + double error_bound_, step_ = 0; + int64_t water_oh2_ = 0, water_hh2_ = 0; // squared O-H and H-H distances of the water (lattice units) + size_t num_waters_ = 0; + int modes_[biomd::NUM_GROUPS] = {1, 0, 0, 0}; // per group: the predictor of the frames after the first + bool unbonded_box_ = false; // one frame, its unbonded atoms as points in their box + int32_t box_min_[3] = {0, 0, 0}; // the box of frame 0: corner + uint32_t box_size_[3] = {1, 1, 1}; // and sides + std::vector raw_bits_; // intra water O's and the low bytes of escaped values +}; + +template +SZBioMDDecomposition make_decomposition_biomd(const Config &conf) { + return SZBioMDDecomposition(conf); +} + +} // namespace SZ3 +#endif diff --git a/include/SZ3/decomposition/TimeSeriesDecomposition.hpp b/include/SZ3/decomposition/TimeSeriesDecomposition.hpp new file mode 100644 index 00000000..a7f78fb9 --- /dev/null +++ b/include/SZ3/decomposition/TimeSeriesDecomposition.hpp @@ -0,0 +1,162 @@ +#ifndef SZ3_TIME_SERIES_DECOMPOSITION_HPP +#define SZ3_TIME_SERIES_DECOMPOSITION_HPP + +#include +#include +#include + +#include "Decomposition.hpp" +#include "SZ3/def.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/BlockwiseIterator.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +namespace SZ3 { +template +class TimeSeriesDecomposition : public concepts::DecompositionInterface { + public: + using Block_iter = typename block_data::block_iterator; + + TimeSeriesDecomposition(const Config& conf, Predictor predictor, Quantizer quantizer, T* data_ts0) + : predictor(predictor), + fallback_predictor(LorenzoPredictor(conf.absErrorBound)), + quantizer(quantizer), + num_elements(conf.num), + data_ts0(data_ts0) { + static_assert(std::is_base_of, Predictor>::value, + "must implement the predictor interface"); + static_assert(std::is_base_of, Quantizer>::value, + "must implement the quantizer interface"); + assert((conf.dims.size() == 2) && "timestep prediction requires 2d dataset"); + } + + ~TimeSeriesDecomposition() = default; + + std::vector compress(const Config& conf, T* data) override { + std::vector quant_inds(num_elements); + size_t quant_count = 0; + // The timestep loop below predicts from the reconstruction of timestep 0. + const T* ts0_recon = data; + std::shared_ptr> data_with_padding; + if (data_ts0 != nullptr) { + for (size_t j = 0; j < conf.dims[1]; j++) { + quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[j], data_ts0[j]); + } + } else { + std::vector spatial_dims(N - 1); + for (uint i = 0; i < N - 1; i++) { + spatial_dims[i] = conf.dims[i + 1]; + }; + + data_with_padding = + std::make_shared>(data, spatial_dims, predictor.get_padding(), true); + auto block = data_with_padding->block_iter(conf.blockSize); + do { + concepts::PredictorInterface* predictor_withfallback = &predictor; + if (!predictor.precompress(block)) { + predictor_withfallback = &fallback_predictor; + } + predictor_withfallback->precompress_block_commit(); + Block_iter::foreach (block, [&](T* c, const std::array& index) { + T pred = predictor_withfallback->predict(block, c, index); + quant_inds[quant_count++] = quantizer.quantize_and_overwrite(*c, pred); + }); + } while (block.next()); + + ts0_recon = data_with_padding->values(); + } + + for (size_t j = 0; j < conf.dims[1]; j++) { + T prev = ts0_recon[j]; + for (size_t i = 1; i < conf.dims[0]; i++) { + size_t idx = i * conf.dims[1] + j; + quant_inds[quant_count++] = quantizer.quantize_and_overwrite(data[idx], prev); + prev = data[idx]; // quantize_and_overwrite left the reconstruction here + } + } + assert(quant_count == num_elements); + quantizer.postcompress_data(); + return quant_inds; + } + + T* decompress(const Config& conf, std::vector& quant_inds, T* dec_data) override { + if (quant_inds.size() < num_elements) { + throw std::out_of_range("SZ3 time series: fewer bins than the grid consumes"); + } + int const* quant_inds_pos = quant_inds.data(); + // std::array intra_block_dims; + // auto dec_data = new T[num_elements]; + + if (data_ts0 != nullptr) { + for (size_t j = 0; j < conf.dims[1]; j++) { + dec_data[j] = quantizer.recover(data_ts0[j], *(quant_inds_pos++)); + } + } else { + std::vector spatial_dims(N - 1); + for (uint i = 0; i < N - 1; i++) { + spatial_dims[i] = conf.dims[i + 1]; + }; + + auto data_with_padding = + std::make_shared>(dec_data, spatial_dims, predictor.get_padding(), false); + auto block = data_with_padding->block_iter(conf.blockSize); + do { + concepts::PredictorInterface* predictor_withfallback = &predictor; + if (!predictor.predecompress(block)) { + predictor_withfallback = &fallback_predictor; + } + Block_iter::foreach (block, [&](T* c, const std::array& index) { + T pred = predictor_withfallback->predict(block, c, index); + *c = quantizer.recover(pred, *(quant_inds_pos++)); + }); + } while (block.next()); + } + + for (size_t j = 0; j < conf.dims[1]; j++) { + for (size_t i = 1; i < conf.dims[0]; i++) { + size_t idx = i * conf.dims[1] + j; + size_t idx_prev = (i - 1) * conf.dims[1] + j; + dec_data[idx] = quantizer.recover(dec_data[idx_prev], *(quant_inds_pos++)); + } + } + + quantizer.postdecompress_data(); + return dec_data; + } + + size_t size_est() override { return fallback_predictor.size_est() + predictor.size_est() + quantizer.size_est(); } + + void save(uchar*& c) override { + fallback_predictor.save(c); + predictor.save(c); + quantizer.save(c); + } + + void load(const uchar*& c, size_t& remaining_length) override { + fallback_predictor.load(c, remaining_length); + predictor.load(c, remaining_length); + quantizer.load(c, remaining_length); + } + + std::pair get_out_range() override { return quantizer.get_out_range(); } + + private: + Predictor predictor; + LorenzoPredictor fallback_predictor; + Quantizer quantizer; + size_t num_elements; + T* data_ts0 = nullptr; +}; + +template +TimeSeriesDecomposition make_decomposition_timeseries(const Config& conf, + Predictor predictor, + Quantizer quantizer, T* data_ts0) { + return TimeSeriesDecomposition(conf, predictor, quantizer, data_ts0); +} +} // namespace SZ3 + +#endif \ No newline at end of file diff --git a/include/SZ3/def.hpp b/include/SZ3/def.hpp new file mode 100644 index 00000000..4d4b3324 --- /dev/null +++ b/include/SZ3/def.hpp @@ -0,0 +1,40 @@ +#ifndef SZ3_DEF_HPP +#define SZ3_DEF_HPP + +#include + +namespace SZ3 { + +typedef unsigned int uint; +typedef unsigned char uchar; +#define SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH \ + "The buffer for compressed data is not large enough." + +#ifdef _MSC_VER +#define ALWAYS_INLINE __forceinline +#elif defined(__GNUC__) || defined(__clang__) +#define ALWAYS_INLINE inline __attribute__((always_inline)) +#else +#define ALWAYS_INLINE inline +#endif + +// In nofma(a * b) + c the product is rounded before the add, so the compiler cannot merge them into one +// FMA instruction, which rounds only once; otherwise a build with FMA and one without decompress different values. +template +ALWAYS_INLINE T nofma(T x) { + // Not __builtin_assoc_barrier: GCC 13 and 14 drop it when they vectorize the loop, and fuse the product anyway. + volatile T r = x; + return r; +} + +// Refuses x87 extended precision and GCC's fast math, which cannot be turned off for part of a program; otherwise +// builds decompress different values. +#if FLT_EVAL_METHOD > 0 +#error "SZ3 does not support x87 floating point; on 32-bit x86 build with -msse2 -mfpmath=sse" +#endif +#if defined(__FAST_MATH__) && !defined(__clang__) +#error "SZ3 does not support GCC's -ffast-math or -Ofast" +#endif +} // namespace SZ3 + +#endif diff --git a/include/SZ3/deprecated/HuffmanEncoderV1.hpp b/include/SZ3/deprecated/HuffmanEncoderV1.hpp new file mode 100644 index 00000000..cf860f93 --- /dev/null +++ b/include/SZ3/deprecated/HuffmanEncoderV1.hpp @@ -0,0 +1,244 @@ +#ifndef SZ3_HUFFMAN_ENCODER_V1_HPP +#define SZ3_HUFFMAN_ENCODER_V1_HPP + +#include +#include +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +namespace SZ3 { + +/// Decodes the Huffman streams of data version 3.3.2 only. +template +class HuffmanEncoderV1 : public concepts::EncoderInterface { + public: + typedef struct node_t { + struct node_t *left, *right; + char t; // in_node:0; otherwise:1 + T c; + } *node; + + typedef struct HuffmanTree { + struct node_t *pool; + int n_nodes; + } HuffmanTree; + + HuffmanEncoderV1() = default; + + ~HuffmanEncoderV1() override { SZ_FreeHuffman(); } + + // Declared because the destructor above suppresses the implicit ones. Copying is shallow and + // has to stay -- the SZAlgo* wirings pass encoders by value -- so do not copy one that has + // built its tree: both copies would free huffmanTree. Today every copy happens before that. + HuffmanEncoderV1(const HuffmanEncoderV1 &) = default; + HuffmanEncoderV1 &operator=(const HuffmanEncoderV1 &) = default; + HuffmanEncoderV1(HuffmanEncoderV1 &&) noexcept = default; + HuffmanEncoderV1 &operator=(HuffmanEncoderV1 &&) noexcept = default; + + HuffmanTree *createHuffmanTree(int stateNum) { + HuffmanTree *tree = static_cast(malloc(sizeof(HuffmanTree))); + tree->pool = static_cast(malloc(4 * static_cast(stateNum) * sizeof(struct node_t))); + memset(tree->pool, 0, 4 * static_cast(stateNum) * sizeof(struct node_t)); + tree->n_nodes = 0; + return tree; + } + + void preprocess_encode(const std::vector &, int) override { throw std::logic_error(kDecodeOnly); } + + void save(uchar *&) override { throw std::logic_error(kDecodeOnly); } + + size_t size_est() override { throw std::logic_error(kDecodeOnly); } + + static size_t size_bound(size_t, size_t) { throw std::logic_error(kDecodeOnly); } + + size_t encode(const std::vector &, uchar *&) override { throw std::logic_error(kDecodeOnly); } + + void postprocess_encode() override { throw std::logic_error(kDecodeOnly); } + + void preprocess_decode() override {} + + // perform decoding + std::vector decode(const uchar *&bytes, size_t targetLength, size_t &remaining_length) override { + node t = treeRoot; + std::vector out(targetLength); + size_t i = 0, byteIndex = 0, count = 0; + int r; + node n = treeRoot; + uint64_t encodedLength = 0; + read(encodedLength, bytes, remaining_length); + if (n->t) // root->t==1 means that all state values are the same (constant) + { + for (count = 0; count < targetLength; count++) out[count] = n->c + offset; + return out; + } + + if (encodedLength > remaining_length) + throw std::out_of_range("SZ3 Huffman: encoded length exceeds compressed buffer"); + + // Walk at most the bits the stream holds, and stop once targetLength symbols are out. + const size_t maxBits = targetLength > 0 ? encodedLength * 8 : 0; + for (i = 0; i < maxBits; i++) { + byteIndex = i >> 3; // i/8 + r = i % 8; + if (((bytes[byteIndex] >> (7 - r)) & 0x01) == 0) + n = n->left; + else + n = n->right; + + if (n->t) { + out[count] = n->c + offset; + n = t; + if (++count == targetLength) break; + } + } + if (count < targetLength) throw std::out_of_range("SZ3 Huffman: corrupted encoded stream"); + bytes += encodedLength; + remaining_length -= encodedLength; + return out; + } + + void postprocess_decode() override { SZ_FreeHuffman(); } + + // load Huffman tree + void load(const uchar *&c, size_t &remaining_length) override { + read(offset, c, remaining_length); + if (remaining_length < 2 * sizeof(int)) throw std::out_of_range("SZ3 Huffman: truncated tree header"); + nodeCount = bytesToInt32_bigEndian(c); + // The stored state count is skipped: it sizes the encode-side code tables, which decoding never + // touches. nodeCount is bounded before it sizes anything, or the encodeStartIndex arithmetic + // below overflows and the tree is read past the buffer. + if (nodeCount <= 0 || static_cast(nodeCount) > remaining_length) + throw std::out_of_range("SZ3 Huffman: invalid node count"); + size_t encodeStartIndex; + if (nodeCount <= 256) + encodeStartIndex = 1 + 3 * nodeCount * sizeof(unsigned char) + nodeCount * sizeof(T); + else if (nodeCount <= 65536) + encodeStartIndex = + 1 + 2 * nodeCount * sizeof(unsigned short) + nodeCount * sizeof(unsigned char) + nodeCount * sizeof(T); + else + encodeStartIndex = + 1 + 2 * nodeCount * sizeof(unsigned int) + nodeCount * sizeof(unsigned char) + nodeCount * sizeof(T); + + size_t tree_bytes = sizeof(int) + sizeof(int) + encodeStartIndex; + if (tree_bytes > remaining_length) throw std::out_of_range("SZ3 Huffman: tree exceeds compressed buffer"); + + // The pool is 4x what is asked for, and unpad_tree builds each of the nodeCount nodes once. + huffmanTree = createHuffmanTree(nodeCount); + treeRoot = reconstruct_HuffTree_from_bytes_anyStates(c + sizeof(int) + sizeof(int), nodeCount); + c += tree_bytes; + remaining_length -= tree_bytes; + } + + private: + static constexpr const char *kDecodeOnly = "SZ3 Huffman: HuffmanEncoderV1 only decodes data version 3.3.2"; + HuffmanTree *huffmanTree = nullptr; + node treeRoot; + unsigned int nodeCount = 0; + T offset; + + node reconstruct_HuffTree_from_bytes_anyStates(const unsigned char *bytes, uint nodeCount_) { + if (nodeCount_ <= 256) { + std::vector L(nodeCount_); + std::vector R(nodeCount_); + std::vector C(nodeCount_); + std::vector t(nodeCount_); + memcpy(L.data(), bytes + 1, nodeCount_ * sizeof(unsigned char)); + memcpy(R.data(), bytes + 1 + nodeCount_ * sizeof(unsigned char), nodeCount_ * sizeof(unsigned char)); + memcpy(C.data(), bytes + 1 + 2 * nodeCount_ * sizeof(unsigned char), nodeCount_ * sizeof(T)); + memcpy(t.data(), bytes + 1 + 2 * nodeCount_ * sizeof(unsigned char) + nodeCount_ * sizeof(T), + nodeCount_ * sizeof(unsigned char)); + std::vector seen(nodeCount_, false); + seen[0] = true; + node root = this->new_node2(C[0], t[0]); + this->unpad_tree(L.data(), R.data(), C.data(), t.data(), 0, root, nodeCount_, seen); + return root; + } else if (nodeCount_ <= 65536) { + std::vector L(nodeCount_); + std::vector R(nodeCount_); + std::vector C(nodeCount_); + std::vector t(nodeCount_); + memcpy(L.data(), bytes + 1, nodeCount_ * sizeof(unsigned short)); + memcpy(R.data(), bytes + 1 + nodeCount_ * sizeof(unsigned short), nodeCount_ * sizeof(unsigned short)); + memcpy(C.data(), bytes + 1 + 2 * nodeCount_ * sizeof(unsigned short), nodeCount_ * sizeof(T)); + memcpy(t.data(), bytes + 1 + 2 * nodeCount_ * sizeof(unsigned short) + nodeCount_ * sizeof(T), + nodeCount_ * sizeof(unsigned char)); + std::vector seen(nodeCount_, false); + seen[0] = true; + node root = this->new_node2(0, 0); + this->unpad_tree(L.data(), R.data(), C.data(), t.data(), 0, root, nodeCount_, seen); + return root; + } else // nodeCount_>65536 + { + std::vector L(nodeCount_); + std::vector R(nodeCount_); + std::vector C(nodeCount_); + std::vector t(nodeCount_); + memcpy(L.data(), bytes + 1, nodeCount_ * sizeof(unsigned int)); + memcpy(R.data(), bytes + 1 + nodeCount_ * sizeof(unsigned int), nodeCount_ * sizeof(unsigned int)); + memcpy(C.data(), bytes + 1 + 2 * nodeCount_ * sizeof(unsigned int), nodeCount_ * sizeof(T)); + memcpy(t.data(), bytes + 1 + 2 * nodeCount_ * sizeof(unsigned int) + nodeCount_ * sizeof(T), + nodeCount_ * sizeof(unsigned char)); + std::vector seen(nodeCount_, false); + seen[0] = true; + node root = this->new_node2(0, 0); + this->unpad_tree(L.data(), R.data(), C.data(), t.data(), 0, root, nodeCount_, seen); + return root; + } + } + + node new_node2(T c, unsigned char t) { + huffmanTree->pool[huffmanTree->n_nodes].c = c; + huffmanTree->pool[huffmanTree->n_nodes].t = t; + return huffmanTree->pool + huffmanTree->n_nodes++; + } + + template + void unpad_tree(T1 *L, T1 *R, T *C, unsigned char *t, unsigned int i, node root, unsigned int nodeCount_, + std::vector &seen) { + if (root->t == 0) { + T1 l, r; + l = L[i]; + if (l != 0) { + // The encoder gave a child a higher index than its parent, so a valid index satisfies i < l < + // nodeCount_. Enforcing it keeps L/R/C/t reads inside the pool and rules out a cycle. + if (l <= i || l >= nodeCount_) throw std::out_of_range("SZ3 Huffman: invalid left child index in tree"); + // Increasing indices rule out a cycle but not two parents naming one child, which would + // expand the tree exponentially instead of building nodeCount_ nodes. + if (seen[l]) throw std::out_of_range("SZ3 Huffman: tree node reached twice"); + seen[l] = true; + node lroot = new_node2(C[l], t[l]); + root->left = lroot; + unpad_tree(L, R, C, t, l, lroot, nodeCount_, seen); + } + r = R[i]; + if (r != 0) { + if (r <= i || r >= nodeCount_) + throw std::out_of_range("SZ3 Huffman: invalid right child index in tree"); + if (seen[r]) throw std::out_of_range("SZ3 Huffman: tree node reached twice"); + seen[r] = true; + node rroot = new_node2(C[r], t[r]); + root->right = rroot; + unpad_tree(L, R, C, t, r, rroot, nodeCount_, seen); + } + if (root->left == nullptr || root->right == nullptr) { + throw std::out_of_range("SZ3 Huffman: internal tree node is missing a child"); + } + } + } + + void SZ_FreeHuffman() { + if (huffmanTree != nullptr) { + free(huffmanTree->pool); + free(huffmanTree); + huffmanTree = nullptr; + } + } +}; +} // namespace SZ3 + +#endif diff --git a/include/SZ3/encoder/BypassEncoder.hpp b/include/SZ3/encoder/BypassEncoder.hpp new file mode 100644 index 00000000..b178fb98 --- /dev/null +++ b/include/SZ3/encoder/BypassEncoder.hpp @@ -0,0 +1,48 @@ +#ifndef SZ3_BYPASS_ENCODER_HPP +#define SZ3_BYPASS_ENCODER_HPP + +#include +#include +#include +#include + +#include "Encoder.hpp" +#include "SZ3/def.hpp" + +namespace SZ3 { + +template +class BypassEncoder : public concepts::EncoderInterface { + public: + void preprocess_encode(const std::vector &bins, int stateNum) override { + } + + size_t encode(const std::vector &bins, uchar *&bytes) override { + memcpy(bytes, &bins[0], sizeof(T) * bins.size()); + bytes += sizeof(T) * bins.size(); + return 0; + } + + void postprocess_encode() override {} + + void preprocess_decode() override {} + + std::vector decode(const uchar *&bytes, size_t targetLength, size_t &remaining_length) override { + if (targetLength > remaining_length / sizeof(T)) { + throw std::out_of_range("SZ3 bypass encoder: more bins requested than the buffer holds"); + } + std::vector bins(targetLength); + memcpy(bins.data(), bytes, sizeof(T) * targetLength); + bytes += sizeof(T) * targetLength; + remaining_length -= sizeof(T) * targetLength; + return bins; + } + + void postprocess_decode() override {} + + void save(uchar *&c) override {} + + void load(const uchar *&c, size_t &remaining_length) override {} +}; +} // namespace SZ3 +#endif diff --git a/include/SZ3/encoder/Encoder.hpp b/include/SZ3/encoder/Encoder.hpp new file mode 100644 index 00000000..caae2757 --- /dev/null +++ b/include/SZ3/encoder/Encoder.hpp @@ -0,0 +1,78 @@ +#ifndef SZ3_ENCODER_HPP +#define SZ3_ENCODER_HPP + +#include +#include + +#include "SZ3/def.hpp" +namespace SZ3 { +namespace concepts { + +/** + * Encoder changes the input to a more compact representative + * Usually this step is lossless instead of lossy + * Examples: huffman, runlenth, etc. + * @tparam T bin type + */ +template > +class EncoderInterface { + public: + virtual ~EncoderInterface() = default; + + /** + * init the encoder + * E.g., Huffman will build tree in this step + * @param bins to-be-encoded integers + * @param stateNum stateNum > 0 indicates the bins has a range of [0, stateNum). stateNum == 0 means no such + * guarantee + */ + virtual void preprocess_encode(const std::vector &bins, int stateNum) = 0; + + /** + * encode the input (in vector format) to a more compact representative(in byte stream format) + * @param bins input in vector + * @param bytes output in byte stream + * @return size of output (# of bytes) + */ + virtual size_t encode(const std::vector &bins, uchar *&bytes) = 0; + + /** + * reverse of encode() + * + * `targetLength` says when to stop producing, `remaining_length` says how far the reads may go; + * an entropy-coded stream has no terminator, so neither number substitutes for the other. + * + * @param bytes input in byte stream + * @param targetLength size of the output vector + * @param remaining_length bytes readable from `bytes`; decremented by what is consumed + * @return output in vector + */ + virtual std::vector decode(const uchar *&bytes, size_t targetLength, size_t &remaining_length) = 0; + + /** + * serialize the encoder and store it to a buffer + * @param c One large buffer is pre-allocated, and the start location of the serialized encoder in the buffer is + * indicated by c. After saving the encoder to the buffer, this function should change c to indicate the next empty + * location in the buffer + */ + virtual void save(uchar *&c) = 0; + + /** + * deserialize the encoder from a buffer + * @param c start location of the encoder in the buffer + * @param remaining_length the remaining length of the buffer + */ + virtual void load(const uchar *&c, size_t &remaining_length) = 0; + + virtual void postprocess_decode() = 0; + + virtual void postprocess_encode() = 0; + + virtual void preprocess_decode() = 0; + + /// Upper bound on the space save() needs for the current input. + virtual size_t size_est() { return 0; } +}; +} // namespace concepts +} // namespace SZ3 +#endif diff --git a/include/SZ3/encoder/HuffmanEncoder.hpp b/include/SZ3/encoder/HuffmanEncoder.hpp new file mode 100644 index 00000000..8dfebb5b --- /dev/null +++ b/include/SZ3/encoder/HuffmanEncoder.hpp @@ -0,0 +1,601 @@ +#ifndef SZ3_HUFFMAN_ENCODER_HPP +#define SZ3_HUFFMAN_ENCODER_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +namespace SZ3 { + +/** + * Canonical Huffman coder for integer bins. + * + * save() writes the number of distinct bins D as a uint32; if D > 0, the smallest bin; if D > 1, for each bin in + * ascending order the Elias-gamma code of its gap from the one before (none for the first), then the Elias-gamma + * code of 1 + the zigzag of its code length minus the previous one's (0 before the first). encode() writes the + * payload's bit count as a uint64, then the codes MSB-first. + * + * Code lengths, capped at 32 bits, are those of a Huffman tree built in (frequency, bin) order, so every platform + * writes the same bytes. + */ +template +class HuffmanEncoder : public concepts::EncoderInterface { + static_assert(std::is_integral::value && sizeof(T) <= 8, "bins must be an integer type of at most 64 bits"); + using U = typename std::make_unsigned::type; + static constexpr unsigned kMaxLen = 32; + static constexpr unsigned kTableBits = 12; + + public: + /// stateNum is ignored: the range is taken from the bins. + void preprocess_encode(const std::vector &bins, int /*stateNum*/) override { + *this = HuffmanEncoder(); + const size_t n = bins.size(); + if (n == 0) return; + // Locals, not offset_: a store to a member of type T may alias the bins and stop the loop vectorising. + T mn = bins[0], mx = bins[0]; + for (size_t i = 0; i < n; i++) { + mn = bins[i] < mn ? bins[i] : mn; + mx = bins[i] > mx ? bins[i] : mx; + } + offset_ = mn; + const uint64_t top = static_cast(static_cast(mx) - static_cast(mn)); + // A table indexed by bin costs its span to clear; past twice the input a hash on the bin is cheaper. + dense_ = top < std::max(4096, 2 * static_cast(n)); + std::vector freq; + if (dense_) { + count_dense(bins, top + 1, freq); + } else { + count_sparse(bins, freq); + } + build_lengths(freq); + build_codes(); + } + + size_t size_est() override { + if (syms_.empty()) return sizeof(uint32_t); + return sizeof(uint32_t) + sizeof(T) + (header_bits_ + 7) / 8; + } + + /// Upper bound of save() plus encode() for num_bins bins holding at most distinct_symbols values. + static size_t size_bound(size_t num_bins, size_t distinct_symbols) { + const size_t d = std::min(num_bins, distinct_symbols); + return sizeof(uint32_t) + sizeof(T) + (d * (16 * sizeof(T) + 10) + 7) / 8 + sizeof(uint64_t) + + (num_bins * kMaxLen + 7) / 8; + } + + void save(uchar *&c) override { + write(static_cast(syms_.size()), c); + if (syms_.empty()) return; + write(offset_, c); + if (syms_.size() == 1) return; + BitWriter w(c); + for (size_t i = 0; i < syms_.size(); i++) { + if (i > 0) w.gamma(syms_[i] - syms_[i - 1]); + w.gamma(zigzag(lens_[i] - (i ? lens_[i - 1] : 0)) + 1); + } + c = w.finish(); + } + + /// bins must be the ones preprocess_encode() saw. + size_t encode(const std::vector &bins, uchar *&bytes) override { + uchar *const start = bytes; + write(payload_bits_, bytes); + if (syms_.size() < 2) { + for (const T b : bins) + if (syms_.empty() || b != offset_) + throw std::invalid_argument("SZ3 Huffman: bin not seen by preprocess_encode"); + return bytes - start; + } + uint64_t acc = 0; + unsigned nb = 0; + uchar *p = bytes; + const U off = static_cast(offset_); + auto put = [&](uint64_t e) { + const unsigned len = static_cast(e & 0xff); + acc = (acc << len) | (e >> 8); + nb += len; + if (nb >= 32) { + nb -= 32; + int32ToBytes_bigEndian(p, static_cast(acc >> nb)); + p += 4; + } + }; + if (dense_) { + const uint64_t *tab = code_.data(); + const uint64_t size = code_.size(); + for (const T b : bins) { + const uint64_t s = static_cast(static_cast(b) - off); + if (s >= size) throw std::invalid_argument("SZ3 Huffman: bin not seen by preprocess_encode"); + put(tab[s]); + } + } else { + for (const T b : bins) put(map_.get(static_cast(static_cast(b) - off))); + } + const uint64_t bits = static_cast(p - bytes) * 8 + nb; + while (nb >= 8) { + nb -= 8; + *p++ = static_cast(acc >> nb); + } + if (nb) *p++ = static_cast(acc << (8 - nb)); + // A bin preprocess_encode() did not count has a zero-length entry. + if (bits != payload_bits_) throw std::invalid_argument("SZ3 Huffman: bins differ from preprocess_encode's"); + bytes = p; + return bytes - start; + } + + void postprocess_encode() override {} + + void preprocess_decode() override {} + + void load(const uchar *&c, size_t &remaining_length) override { + *this = HuffmanEncoder(); + const uchar *p = c; + size_t rem = remaining_length; + uint32_t d = 0; + read(d, p, rem); + if (d > 0) read(offset_, p, rem); + if (d == 1) { + syms_.assign(1, 0); + lens_.assign(1, 0); + } else if (d > 1) { + // Every bin after the first takes at least two bits of the table. + if (d - 1 > rem * 4) throw std::out_of_range("SZ3 Huffman: code table exceeds the buffer"); + BitReader r(p, rem); + read_table(r, d); + const size_t used = r.bytes(); + p += used; + rem -= used; + } + build_decoder(); + remaining_length = rem; + c = p; + } + + std::vector decode(const uchar *&bytes, size_t targetLength, size_t &remaining_length) override { + const uchar *p = bytes; + size_t rem = remaining_length; + uint64_t bits = 0; + read(bits, p, rem); + if (bits > static_cast(rem) * 8) throw std::out_of_range("SZ3 Huffman: payload exceeds the buffer"); + const size_t nbytes = static_cast((bits + 7) / 8); + std::vector out; + if (syms_.size() < 2) { + if (bits != 0 || (syms_.empty() && targetLength != 0)) + throw std::out_of_range("SZ3 Huffman: payload does not match the code table"); + out.assign(targetLength, offset_); + } else { + if (bits < targetLength || bits / kMaxLen > targetLength) + throw std::out_of_range("SZ3 Huffman: payload does not match the value count"); + out.resize(targetLength); + // A one-bit code is all zeros, so a run of zero bits is a run of its symbol. + if (count_[1]) + decode_payload(p, nbytes, bits, out.data(), targetLength); + else + decode_payload(p, nbytes, bits, out.data(), targetLength); + } + bytes = p + nbytes; + remaining_length = rem - nbytes; + return out; + } + + void postprocess_decode() override {} + + private: + class BitWriter { + public: + explicit BitWriter(uchar *p) : p_(p) {} + void put(uint64_t v, unsigned n) { // n <= 32 + acc_ = (acc_ << n) | (v & ((uint64_t(1) << n) - 1)); + nb_ += n; + while (nb_ >= 8) { + nb_ -= 8; + *p_++ = static_cast(acc_ >> nb_); + } + } + void gamma(uint64_t v) { // v >= 1 + unsigned n = 64 - clz64(v); + for (unsigned z = n - 1; z > 0; z -= std::min(z, 32u)) put(0, std::min(z, 32u)); + if (n > 32) { + put(v >> 32, n - 32); + n = 32; + } + put(v, n); + } + uchar *finish() { + if (nb_) *p_++ = static_cast(acc_ << (8 - nb_)); + return p_; + } + + private: + uchar *p_; + uint64_t acc_ = 0; + unsigned nb_ = 0; + }; + + class BitReader { + public: + BitReader(const uchar *p, size_t n) : p_(p), n_(n) {} + unsigned bit() { + if (pos_ >= n_ * 8) throw std::out_of_range("SZ3 Huffman: truncated code table"); + const unsigned b = (p_[pos_ >> 3] >> (7 - (pos_ & 7))) & 1; + pos_++; + return b; + } + uint64_t gamma() { + unsigned z = 0; + while (bit() == 0) + if (++z > 63) throw std::out_of_range("SZ3 Huffman: invalid gamma code"); + uint64_t v = 1; + for (unsigned i = 0; i < z; i++) v = (v << 1) | bit(); + return v; + } + size_t bytes() const { return (pos_ + 7) / 8; } + + private: + const uchar *p_; + size_t n_; + size_t pos_ = 0; + }; + + static unsigned clz64(uint64_t v) { // v != 0 +#if defined(__GNUC__) || defined(__clang__) + return static_cast(__builtin_clzll(v)); +#else + unsigned n = 0; + for (unsigned s = 32; s > 0; s >>= 1) + if (!(v >> (64 - s))) { + n += s; + v <<= s; + } + return n; +#endif + } + + static uint64_t gamma_bits(uint64_t v) { return 2 * (64 - clz64(v)) - 1; } + + static uint64_t zigzag(int d) { return d >= 0 ? 2 * static_cast(d) : 2 * static_cast(-d) - 1; } + + void read_table(BitReader &r, size_t d) { + syms_.resize(d); + lens_.resize(d); + // offset_ + bin must stay a T. + const uint64_t max_sym = + static_cast(static_cast(std::numeric_limits::max()) - static_cast(offset_)); + uint64_t s = 0; + int prev = 0; + for (size_t i = 0; i < d; i++) { + if (i > 0) { + const uint64_t g = r.gamma(); + if (g > max_sym - s) throw std::out_of_range("SZ3 Huffman: bin out of range"); + s += g; + } + const uint64_t z = r.gamma() - 1; + if (z > 2 * kMaxLen) throw std::out_of_range("SZ3 Huffman: invalid code length"); + const int len = prev + ((z & 1) ? -static_cast((z + 1) / 2) : static_cast(z / 2)); + if (len < 1 || len > static_cast(kMaxLen)) throw std::out_of_range("SZ3 Huffman: invalid code length"); + syms_[i] = s; + lens_[i] = static_cast(len); + prev = len; + } + } + + void count_dense(const std::vector &bins, uint64_t span, std::vector &freq_out) { + const U off = static_cast(offset_); + const size_t n = bins.size(); + std::vector f64; + // Runs of one bin make one counter a store-to-load chain; four counters break it while they stay in cache. + if (span * sizeof(uint32_t) * 4 <= (size_t(8) << 20) && n >= 4096 && n <= UINT32_MAX) { + std::vector f32(span * 4, 0); + uint32_t *f0 = f32.data(), *f1 = f0 + span, *f2 = f1 + span, *f3 = f2 + span; + size_t i = 0; + for (; i + 4 <= n; i += 4) { + f0[static_cast(static_cast(bins[i]) - off)]++; + f1[static_cast(static_cast(bins[i + 1]) - off)]++; + f2[static_cast(static_cast(bins[i + 2]) - off)]++; + f3[static_cast(static_cast(bins[i + 3]) - off)]++; + } + for (; i < n; i++) f0[static_cast(static_cast(bins[i]) - off)]++; + f64.resize(span); + for (size_t k = 0; k < span; k++) f64[k] = uint64_t(f0[k]) + f1[k] + f2[k] + f3[k]; + } else { + f64.assign(span, 0); + for (const T b : bins) f64[static_cast(static_cast(b) - off)]++; + } + for (size_t k = 0; k < span; k++) { + if (f64[k]) { + syms_.push_back(k); + freq_out.push_back(f64[k]); + } + } + code_.assign(span, 0); + } + + // Open addressing on bin - offset, hashed so that evenly spaced bins do not pile up in neighbouring slots. + struct SymMap { + std::vector key, val; + std::vector used; + size_t count = 0; + unsigned shift = 64; + + size_t slot(uint64_t k) const { + k = (k ^ (k >> 30)) * 0xbf58476d1ce4e5b9ull; + k = (k ^ (k >> 27)) * 0x94d049bb133111ebull; + return static_cast((k ^ (k >> 31)) >> shift); + } + uint64_t &at(uint64_t k) { // inserts + if (2 * (count + 1) > key.size()) grow(); + const size_t m = key.size() - 1; + size_t i = slot(k); + while (used[i] && key[i] != k) i = (i + 1) & m; + if (!used[i]) { + used[i] = 1; + key[i] = k; + count++; + } + return val[i]; + } + uint64_t get(uint64_t k) const { // 0 if absent + const size_t m = key.size() - 1; + for (size_t i = slot(k); used[i]; i = (i + 1) & m) + if (key[i] == k) return val[i]; + return 0; + } + void grow() { + const unsigned bits = key.empty() ? 10 : 65 - shift; + SymMap bigger; + bigger.key.assign(size_t(1) << bits, 0); + bigger.val.assign(size_t(1) << bits, 0); + bigger.used.assign(size_t(1) << bits, 0); + bigger.shift = 64 - bits; + for (size_t i = 0; i < key.size(); i++) + if (used[i]) bigger.at(key[i]) = val[i]; + *this = std::move(bigger); + } + }; + + void count_sparse(const std::vector &bins, std::vector &freq_out) { + const U off = static_cast(offset_); + for (const T b : bins) map_.at(static_cast(static_cast(b) - off))++; + std::vector> kv; + kv.reserve(map_.count); + for (size_t i = 0; i < map_.key.size(); i++) + if (map_.used[i]) kv.emplace_back(map_.key[i], map_.val[i]); + std::sort(kv.begin(), kv.end()); + for (const auto &e : kv) { + syms_.push_back(e.first); + freq_out.push_back(e.second); + } + } + + void build_lengths(const std::vector &freq) { + const size_t d = syms_.size(); + if (d > UINT32_MAX) throw std::invalid_argument("SZ3 Huffman: more distinct bins than a uint32 counts"); + lens_.assign(d, 0); + if (d == 1) return; + // Ascending frequency, ties by bin, so the lengths do not depend on the platform's sort. + std::vector order(d); + for (size_t i = 0; i < d; i++) order[i] = static_cast(i); + std::sort(order.begin(), order.end(), + [&](uint32_t a, uint32_t b) { return freq[a] != freq[b] ? freq[a] < freq[b] : a < b; }); + std::vector A(d); + for (size_t i = 0; i < d; i++) A[i] = freq[order[i]]; + minimum_redundancy(A); + + // Clamp to kMaxLen, then move the deepest leaves shorter than kMaxLen down until Kraft's inequality holds. + uint64_t count[kMaxLen + 1] = {0}; + for (size_t i = 0; i < d; i++) count[std::min(A[i], kMaxLen)]++; + uint64_t kraft = 0; + for (unsigned l = 1; l <= kMaxLen; l++) kraft += count[l] << (kMaxLen - l); + while (kraft > (uint64_t(1) << kMaxLen)) { + unsigned l = kMaxLen - 1; + while (count[l] == 0) l--; + count[l]--; + count[l + 1]++; + kraft -= uint64_t(1) << (kMaxLen - l - 1); + } + size_t i = 0; + for (unsigned l = kMaxLen; l >= 1; l--) + for (uint64_t k = 0; k < count[l]; k++, i++) { + lens_[order[i]] = static_cast(l); + payload_bits_ += freq[order[i]] * l; + } + header_bits_ = 0; + for (size_t j = 0; j < d; j++) { + if (j > 0) header_bits_ += gamma_bits(syms_[j] - syms_[j - 1]); + header_bits_ += gamma_bits(zigzag(lens_[j] - (j ? lens_[j - 1] : 0)) + 1); + } + } + + // Moffat and Katajainen, "In-place calculation of minimum-redundancy codes" (1995): A holds the weights in + // ascending order and receives the code lengths. + static void minimum_redundancy(std::vector &A) { + const size_t n = A.size(); + size_t root = 0, leaf = 2; + A[0] += A[1]; + for (size_t next = 1; next < n - 1; next++) { + if (leaf >= n || A[root] < A[leaf]) { + A[next] = A[root]; + A[root++] = next; + } else { + A[next] = A[leaf++]; + } + if (leaf >= n || (root < next && A[root] < A[leaf])) { + A[next] += A[root]; + A[root++] = next; + } else { + A[next] += A[leaf++]; + } + } + A[n - 2] = 0; + for (size_t k = n - 2; k-- > 0;) A[k] = A[A[k]] + 1; + size_t avbl = 1, used = 0, depth = 0, next = n; + size_t r = n - 1; // one past the internal node to look at + while (avbl > 0) { + while (r > 0 && A[r - 1] == depth) { + used++; + r--; + } + while (avbl > used) { + A[--next] = depth; + avbl--; + } + avbl = 2 * used; + depth++; + used = 0; + } + } + + // Codes of one length are consecutive and follow bin order. + void canonical(uint64_t first[kMaxLen + 1]) { + std::fill(count_, count_ + kMaxLen + 1, 0); + for (const uint8_t l : lens_) count_[l]++; + count_[0] = 0; + uint64_t code = 0; + for (unsigned l = 1; l <= kMaxLen; l++) { + first[l] = code; + code = (code + count_[l]) << 1; + } + } + + void build_codes() { + if (syms_.size() < 2) return; + uint64_t first[kMaxLen + 1]; + canonical(first); + for (size_t i = 0; i < syms_.size(); i++) { + const uint64_t e = (first[lens_[i]]++ << 8) | lens_[i]; + if (dense_) + code_[syms_[i]] = e; + else + map_.at(syms_[i]) = e; + } + } + + struct Entry { + T sym; + uint8_t len; // 0: the code is longer than the table, or is not a code + }; + + void build_decoder() { + const size_t d = syms_.size(); + if (d < 2) return; + canonical(first_); + uint64_t kraft = 0; + for (unsigned l = 1; l <= kMaxLen; l++) { + kraft += count_[l] << (kMaxLen - l); + if (count_[l]) max_len_ = l; + } + if (kraft > (uint64_t(1) << kMaxLen)) throw std::out_of_range("SZ3 Huffman: code lengths overfull"); + base_[1] = 0; + for (unsigned l = 2; l <= kMaxLen; l++) base_[l] = base_[l - 1] + count_[l - 1]; + // Codes of length l are below limit_[l] once left-aligned in 32 bits. + for (unsigned l = 1; l <= kMaxLen; l++) limit_[l] = (first_[l] + count_[l]) << (kMaxLen - l); + uint64_t pos[kMaxLen + 1]; + std::copy(base_, base_ + kMaxLen + 1, pos); + sorted_.resize(d); + for (size_t i = 0; i < d; i++) sorted_[pos[lens_[i]]++] = static_cast(static_cast(offset_) + syms_[i]); + table_bits_ = std::min(max_len_, kTableBits); + table_.assign(size_t(1) << table_bits_, Entry{0, 0}); + for (unsigned l = 1; l <= table_bits_; l++) + for (uint64_t k = 0; k < count_[l]; k++) { + const size_t lo = static_cast((first_[l] + k) << (table_bits_ - l)); + std::fill(table_.begin() + lo, table_.begin() + lo + (size_t(1) << (table_bits_ - l)), + Entry{sorted_[base_[l] + k], static_cast(l)}); + } + } + + // A code longer than the table, with its bits at the top of acc. + void decode_long(uint64_t acc, T &sym, unsigned &len) const { + // The length is the first whose limit lies above the code's top 32 bits; counting, not searching, avoids a + // mispredicted branch per length. + const uint64_t w = acc >> 32; + len = table_bits_ + 1; + for (unsigned l = table_bits_ + 1; l < max_len_; l++) len += w >= limit_[l]; + if (w >= limit_[len]) throw std::out_of_range("SZ3 Huffman: invalid code in payload"); + sym = sorted_[base_[len] + ((w >> (32 - len)) - first_[len])]; + } + + template + void decode_payload(const uchar *p, size_t nbytes, uint64_t bits, T *out, size_t n) const { + const Entry *tab = table_.data(); + const unsigned tb = table_bits_, ml = max_len_; + const T zero_sym = sorted_[0]; + // With two one-bit codes, a run of ones is a run of the second symbol. + const bool one_runs = count_[1] == 2; + uint64_t pos = 0; + size_t i = 0; + // Whole 8-byte reads while they stay inside the payload; each gives at least 57 bits. + while (i < n && (pos >> 3) + 8 <= nbytes) { + uint64_t acc = static_cast(bytesToInt64_bigEndian(p + (pos >> 3))) << (pos & 7); + unsigned avail = 64 - static_cast(pos & 7); + do { + unsigned len; + // Only runs of eight or more, which are likely long; shorter ones would mispredict each time. + const bool ones = ZeroRuns && one_runs && (acc >> 56) == 0xff; + if (ZeroRuns && (!(acc >> 56) || ones)) { + const uint64_t w = ones ? ~acc : acc; + len = w ? clz64(w) : 64; + len = static_cast(std::min(std::min(len, avail), n - i)); + std::fill(out + i, out + i + len, ones ? sorted_[1] : zero_sym); + acc = len < 64 ? acc << len : 0; + i += len; + } else { + const Entry e = tab[acc >> (64 - tb)]; + len = e.len; + if (len) + out[i] = e.sym; + else + decode_long(acc, out[i], len); + acc <<= len; + i++; + } + avail -= len; + pos += len; + } while (avail >= ml && i < n); + } + while (i < n && pos <= bits) { + uchar tail[8] = {0}; + memcpy(tail, p + (pos >> 3), std::min(8, nbytes - (pos >> 3))); + const uint64_t acc = static_cast(bytesToInt64_bigEndian(tail)) << (pos & 7); + const Entry e = tab[acc >> (64 - tb)]; + unsigned len = e.len; + if (len) + out[i] = e.sym; + else + decode_long(acc, out[i], len); + pos += len; + i++; + } + if (pos != bits) throw std::out_of_range("SZ3 Huffman: payload length does not match its codes"); + } + + // encoder and decoder + T offset_ = 0; + std::vector syms_; // distinct bin - offset_, ascending + std::vector lens_; // code length of each + uint64_t header_bits_ = 0; + uint64_t payload_bits_ = 0; + uint64_t count_[kMaxLen + 1] = {0}; // codes of each length + // encoder + bool dense_ = true; + std::vector code_; // dense: (code << 8) | length, indexed by bin - offset_ + SymMap map_; // sparse: the same, keyed by bin - offset_ + // decoder + unsigned max_len_ = 0, table_bits_ = 0; + uint64_t first_[kMaxLen + 1] = {0}, base_[kMaxLen + 1] = {0}, limit_[kMaxLen + 1] = {0}; + std::vector sorted_; // bins in code order + std::vector table_; +}; + +} // namespace SZ3 + +#endif diff --git a/include/SZ3/encoder/SegmentedEncoder.hpp b/include/SZ3/encoder/SegmentedEncoder.hpp new file mode 100644 index 00000000..a56fcdfd --- /dev/null +++ b/include/SZ3/encoder/SegmentedEncoder.hpp @@ -0,0 +1,132 @@ +#ifndef SZ3_SEGMENTED_ENCODER_HPP +#define SZ3_SEGMENTED_ENCODER_HPP + +#include +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/encoder/Encoder.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +namespace SZ3 { + +/** + * Bins that are a sequence of segments [n, n bins]: each segment gets its own Encoder, so a decomposition that + * concatenates streams of different statistics gets a code per stream. Decoding returns the bins as they were. + * + * save() writes the segment count as a uint32, then for each segment its size as a LEB128 varint (at most that of + * INT_MAX) and, unless it is empty, its encoder's save(); encode() writes the encoders' encode() in segment order. + * A stream with more segments than max_segments is refused on load(), before an encoder is built for each. + */ +template +class SegmentedEncoder : public concepts::EncoderInterface { + public: + explicit SegmentedEncoder(uint32_t max_segments) : max_segments_(max_segments) {} + + void preprocess_encode(const std::vector &bins, int stateNum) override { + segments_.clear(); + encoders_.clear(); + for (size_t k = 0; k < bins.size();) { + const size_t n = size_t(bins[k]); + if (bins[k] < 0 || n > bins.size() - k - 1) + throw std::invalid_argument( + "SZ3 SegmentedEncoder: bad input format, a segment length is negative or runs past the end"); + segments_.emplace_back(bins.begin() + k + 1, bins.begin() + k + 1 + n); + if (n) { + encoders_.emplace_back(); + encoders_.back().preprocess_encode(segments_.back(), stateNum); + } + k += n + 1; + } + } + + size_t encode(const std::vector & /*bins*/, uchar *&bytes) override { + uchar *const start = bytes; + for (size_t s = 0, e = 0; s < segments_.size(); s++) + if (!segments_[s].empty()) encoders_[e++].encode(segments_[s], bytes); + return bytes - start; + } + + void postprocess_encode() override { + for (auto &encoder : encoders_) encoder.postprocess_encode(); + segments_.clear(); + } + + void preprocess_decode() override { + for (auto &encoder : encoders_) encoder.preprocess_decode(); + } + + std::vector decode(const uchar *&bytes, size_t targetLength, size_t &remaining_length) override { + // a count and the bins of each segment; checked before anything is allocated + uint64_t total = segment_sizes_.size(); + for (const uint64_t n : segment_sizes_) { + if (total > targetLength || n > targetLength - total) + throw std::out_of_range( + "SZ3 SegmentedEncoder: corrupt stream, the segment lengths do not add up to the bin count"); + total += n; + } + if (total != targetLength) + throw std::out_of_range( + "SZ3 SegmentedEncoder: corrupt stream, the segment lengths do not add up to the bin count"); + std::vector out; + out.reserve(targetLength); + for (size_t s = 0, e = 0; s < segment_sizes_.size(); s++) { + out.push_back(int(segment_sizes_[s])); + if (!segment_sizes_[s]) continue; + const auto v = encoders_[e++].decode(bytes, segment_sizes_[s], remaining_length); + out.insert(out.end(), v.begin(), v.end()); + } + return out; + } + + void postprocess_decode() override { + for (auto &encoder : encoders_) encoder.postprocess_decode(); + } + + void save(uchar *&c) override { + write(uint32_t(segments_.size()), c); + for (size_t s = 0, e = 0; s < segments_.size(); s++) { + write_varint(segments_[s].size(), c); + if (!segments_[s].empty()) encoders_[e++].save(c); + } + } + + void load(const uchar *&c, size_t &remaining_length) override { + uint32_t count = 0; + read(count, c, remaining_length); + if (count > remaining_length) + throw std::out_of_range("SZ3 SegmentedEncoder: corrupt stream, more segments than bytes"); + if (count > max_segments_) + throw std::out_of_range("SZ3 SegmentedEncoder: corrupt stream, more segments than the data has"); + // an encoder per non-empty segment as it is read, so memory follows the stream rather than the count it claims + segment_sizes_.clear(); + encoders_.clear(); + for (uint32_t s = 0; s < count; s++) { + const uint64_t size = read_varint(c, remaining_length); + if (size > uint64_t(std::numeric_limits::max())) + throw std::out_of_range("SZ3 SegmentedEncoder: corrupt stream, a segment length beyond INT_MAX"); + segment_sizes_.push_back(size); + if (size) { + encoders_.emplace_back(); + encoders_.back().load(c, remaining_length); + } + } + } + + size_t size_est() override { + size_t bound = sizeof(uint32_t) + 5 * segments_.size(); // 5: a varint of 31 bits + for (auto &encoder : encoders_) bound += encoder.size_est(); + return bound; + } + + private: + std::vector encoders_; // one per non-empty segment + std::vector> segments_; + std::vector segment_sizes_; + uint32_t max_segments_; +}; + +} // namespace SZ3 +#endif diff --git a/include/SZ3/lossless/Lossless.hpp b/include/SZ3/lossless/Lossless.hpp new file mode 100644 index 00000000..5bb77d3e --- /dev/null +++ b/include/SZ3/lossless/Lossless.hpp @@ -0,0 +1,45 @@ +// +// Created by Kai Zhao on 6/12/20. +// + +#ifndef SZ3_LOSSLESS_HPP +#define SZ3_LOSSLESS_HPP + +#include + +#include "SZ3/def.hpp" + +namespace SZ3::concepts { + +/** + * Lossless compressors is used in addition to lossy compression to further reduce the data size + * Usually this module calls into existing lossless compress APIs, instead of re-implementing the lossless algorithms. + */ +class LosslessInterface { + public: + virtual ~LosslessInterface() = default; + + /** + * compress data with lossless compressors + * @param src data to be compressed + * @param srcLen length (in bytes) of the data to be compressed + * @param dst compressed data + * @param dstCap capacity (in bytes) for storing the compressed data + * @return length (in bytes) of the data compressed + */ + virtual size_t compress(const uchar *src, size_t srcLen, uchar *dst, size_t dstCap) = 0; + + /** + * reverse of compress(), decompress the data with lossless compressors + * @param src data to be decompressed + * @param srcLen length (in bytes) of that data + * @param dst buffer to decompress into; when null on entry the callee allocates it with malloc() + * and the caller frees it + * @param dstCap the capacity of dst, ignored when dst is null + * @return length (in bytes) of the data decompressed + */ + virtual size_t decompress(const uchar *src, size_t srcLen, uchar *&dst, size_t dstCap) = 0; +}; +} // namespace SZ3::concepts + +#endif // SZ_LOSSLESS_HPP diff --git a/include/SZ3/lossless/Lossless_bypass.hpp b/include/SZ3/lossless/Lossless_bypass.hpp new file mode 100644 index 00000000..9121834e --- /dev/null +++ b/include/SZ3/lossless/Lossless_bypass.hpp @@ -0,0 +1,57 @@ +// +// Created by Kai Zhao on 4/21/20. +// + +#ifndef SZ3_LOSSLESS_BYPASS_HPP +#define SZ3_LOSSLESS_BYPASS_HPP + +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/lossless/Lossless.hpp" + +namespace SZ3 { +class Lossless_bypass : public concepts::LosslessInterface { +public: + /** + * compress data with lossless compressors + * @param src data to be compressed + * @param srcLen length (in bytes) of the data to be compressed + * @param dst compressed data + * @param dstCap capacity (in bytes) for storing the compressed data + * @return length (in bytes) of the data compressed + */ + size_t compress(const uchar *src, size_t srcLen, uchar *dst, size_t dstCap) override { + if (dstCap < srcLen) { + throw std::length_error(SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH); + } + std::memcpy(dst, src, srcLen); + return srcLen; + } + + /** + * reverse of compress(), decompress the data with lossless compressors + * @param src data to be decompressed + * @param srcLen length (in bytes) of that data + * @param dst buffer to decompress into; when null on entry the callee allocates it with malloc() + * and the caller frees it + * @param dstCap the capacity of dst, ignored when dst is null + * @return length (in bytes) of the data decompressed + */ + size_t decompress(const uchar *src, size_t srcLen, uchar *&dst, size_t dstCap) override { + // malloc, because the caller frees what it gets back with free(). + if (dst == nullptr) { + dst = static_cast(malloc(srcLen)); + if (dst == nullptr) { + throw std::runtime_error("SZ3 bypass lossless: can not allocate the decompression buffer"); + } + } else if (srcLen > dstCap) { + throw std::out_of_range("SZ3 bypass lossless: payload exceeds the allowed capacity"); + } + std::memcpy(dst, src, srcLen); + return srcLen; + } +}; +} // namespace SZ3 +#endif // SZ_LOSSLESS_BYPASS_HPP diff --git a/include/SZ3/lossless/Lossless_zstd.hpp b/include/SZ3/lossless/Lossless_zstd.hpp new file mode 100644 index 00000000..fde13e0f --- /dev/null +++ b/include/SZ3/lossless/Lossless_zstd.hpp @@ -0,0 +1,101 @@ +// +// Created by Kai Zhao on 4/21/20. +// + +#ifndef SZ3_LOSSLESS_ZSTD_HPP +#define SZ3_LOSSLESS_ZSTD_HPP + +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/lossless/Lossless.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +// Do not include here: these headers are installed, so every consumer would need it too. +#ifdef SZ3_USE_ZSTD_HEADER +#include +#else +#include +extern "C" { +size_t ZSTD_compress(void *dst, size_t dstCapacity, const void *src, size_t srcSize, int compressionLevel); +size_t ZSTD_decompress(void *dst, size_t dstCapacity, const void *src, size_t compressedSize); +size_t ZSTD_compressBound(size_t srcSize); +unsigned ZSTD_isError(size_t code); +} +#endif + +namespace SZ3 { +class Lossless_zstd : public concepts::LosslessInterface { + public: + Lossless_zstd() = default; + + Lossless_zstd(int comp_level) : compression_level(comp_level) {} + + /** ZSTD_compressBound is declared only in this header, so callers elsewhere go through this. */ + static size_t compress_bound(size_t srcLen) { return ZSTD_compressBound(srcLen); } + + /** + * compress data with lossless compressors + * @param src data to be compressed + * @param srcLen length (in bytes) of the data to be compressed + * @param dst compressed data + * @param dstCap capacity (in bytes) for storing the compressed data + * @return length (in bytes) of the data compressed + */ + size_t compress(const uchar *src, size_t srcLen, uchar *dst, size_t dstCap) override { + if (dstCap < sizeof(uint64_t)) throw std::length_error(SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH); + write(srcLen, dst); + dstCap -= sizeof(uint64_t); // reserve space for srcLen + size_t dstLen = ZSTD_compress(dst, dstCap, src, srcLen, compression_level); + if (ZSTD_isError(dstLen)) { + throw std::length_error(SZ3_ERROR_COMP_BUFFER_NOT_LARGE_ENOUGH); + } + return dstLen + sizeof(uint64_t); + } + + /** + * reverse of compress(), decompress the data with lossless compressors + * @param src data to be decompressed + * @param srcLen length (in bytes) of that data + * @param dst buffer to decompress into; when null on entry the callee allocates it with malloc() + * and the caller frees it + * @param dstCap the capacity of dst, ignored when dst is null + * @return length (in bytes) of the data decompressed + */ + size_t decompress(const uchar *src, size_t srcLen, uchar *&dst, size_t dstCap) override { + // The stream is a decompressed-size field followed by the zstd frame, all untrusted. + if (srcLen < sizeof(uint64_t)) { + throw std::out_of_range("SZ3 lossless: compressed data is smaller than the size header"); + } + uint64_t dstLen = 0; + read(dstLen, src); + + // malloc, because the caller frees what it gets back with free(). + std::unique_ptr owner(nullptr, &free); + if (dst == nullptr) { + owner.reset(static_cast(malloc(dstLen))); + if (owner == nullptr) { + throw std::runtime_error("SZ3 lossless: can not allocate the decompression buffer"); + } + } else if (dstLen > dstCap) { + throw std::out_of_range("SZ3 lossless: declared decompressed size exceeds the allowed capacity"); + } + uchar *out = (dst != nullptr) ? dst : owner.get(); + + // A short frame would leave the tail of the output uninitialized for the caller to read. + size_t res = ZSTD_decompress(out, dstLen, src, srcLen - sizeof(uint64_t)); + if (ZSTD_isError(res) || res != dstLen) { + throw std::runtime_error("SZ3 lossless: stream does not decompress to the size it declares"); + } + + dst = out; + owner.release(); + return res; + } + + private: + int compression_level = 3; // default setting of level is 3 +}; +} // namespace SZ3 +#endif // SZ_LOSSLESS_ZSTD_HPP diff --git a/include/SZ3/predictor/ComposedPredictor.hpp b/include/SZ3/predictor/ComposedPredictor.hpp new file mode 100644 index 00000000..d34992d7 --- /dev/null +++ b/include/SZ3/predictor/ComposedPredictor.hpp @@ -0,0 +1,135 @@ +#ifndef SZ3_COMPOSED_PREDICTOR_HPP +#define SZ3_COMPOSED_PREDICTOR_HPP + +#include +#include +#include +#include + +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/predictor/Predictor.hpp" + +namespace SZ3 { + +template > +class ComposedPredictor : public concepts::PredictorInterface { + public: + using block_iter = typename block_data::block_iterator; + + ComposedPredictor(std::vector>> predictors_) + : predictors(predictors_) { + if (predictors.empty()) { + throw std::invalid_argument("Empty predictor list for ComposedPredictor."); + } + } + + bool precompress(const block_iter &block) override { + std::vector predict_error(predictors.size(), 0); + std::vector isvalid(predictors.size()); + for (size_t i = 0; i < predictors.size(); i++) { + isvalid[i] = predictors[i]->precompress(block); + if (isvalid[i]) { + block_iter::foreach_sampling(block, [&](T *c, const std::array &index) { + predict_error[i] += predictors[i]->estimate_error(block, c, index); + }); + } else { + predict_error[i] = std::numeric_limits::max(); + } + } + sid = std::distance(predict_error.begin(), std::min_element(predict_error.begin(), predict_error.end())); + return isvalid[sid]; + } + + void precompress_block_commit() override { + selection.push_back(sid); + predictors[sid]->precompress_block_commit(); + } + + bool predecompress(const block_iter &block) override { + // selection and its entries are untrusted, and predict()/estimate_error() reuse the sid set here. + if (current_index >= selection.size()) + throw std::out_of_range("SZ3: ran out of predictor selections while decompressing"); + sid = selection[current_index++]; + if (sid < 0 || static_cast(sid) >= predictors.size()) + throw std::out_of_range("SZ3: predictor selection index is out of range"); + return predictors[sid]->predecompress(block); + } + + size_t size_est() override { + size_t total = sizeof(uint64_t); + for (const auto &p : predictors) { + total += p->size_est(); + } + if (!selection.empty()) { + total += Encoder::size_bound(selection.size(), predictors.size()); + } + return total; + } + + void save(uchar *&c) override { + for (const auto &p : predictors) { + p->save(c); + } + write(selection.size(), c); + if (selection.size() > 0) { + Encoder selection_encoder; + selection_encoder.preprocess_encode(selection, predictors.size()); + selection_encoder.save(c); + selection_encoder.encode(selection, c); + selection_encoder.postprocess_encode(); + } + } + + void load(const uchar *&c, size_t &remaining_length) override { + for (const auto &p : predictors) { + p->load(c, remaining_length); + } + uint64_t selection_size = 0; + read(selection_size, c, remaining_length); + if (selection_size > 0) { + Encoder selection_encoder; + selection_encoder.load(c, remaining_length); + this->selection = selection_encoder.decode(c, selection_size, remaining_length); + selection_encoder.postprocess_decode(); + } + } + + T predict(const block_iter &block, T *d, const std::array &index) override { + return predictors[sid]->predict(block, d, index); + } + + T estimate_error(const block_iter &block, T *d, const std::array &index) override { + return predictors[sid]->estimate_error(block, d, index); + } + + void print() const override { + std::vector cnt(predictors.size(), 0); + size_t cnt_total = 0; + for (auto &sel : selection) { + cnt[sel]++; + cnt_total++; + } + for (size_t i = 0; i < predictors.size(); i++) { + // predictors[i]->print(); + printf("Blocks:%zu, Percentage:%.2f\n", cnt[i], 1.0 * cnt[i] / cnt_total); + } + } + + size_t get_padding() override { + size_t max_padding = 0; + for (const auto &p : predictors) { + max_padding = std::max(max_padding, p->get_padding()); + } + return max_padding; + } + + private: + std::vector>> predictors; + std::vector selection; + int sid = 0; // selected index + size_t current_index = 0; // for decompression only +}; + +} // namespace SZ3 + +#endif diff --git a/include/SZ3/predictor/LorenzoPredictor.hpp b/include/SZ3/predictor/LorenzoPredictor.hpp new file mode 100644 index 00000000..223e59d0 --- /dev/null +++ b/include/SZ3/predictor/LorenzoPredictor.hpp @@ -0,0 +1,118 @@ +#ifndef SZ3_LORENZO_PREDICTOR_HPP +#define SZ3_LORENZO_PREDICTOR_HPP + +#include +#include +#include + +#include "SZ3/predictor/Predictor.hpp" + +namespace SZ3 { + +// N-dimension L-layer lorenzo predictor +template +class LorenzoPredictor : public concepts::PredictorInterface { + static_assert(std::is_floating_point::value, "LorenzoPredictor only takes a floating-point T"); + + public: + static const uint8_t predictor_id = 0b00000001; + using block_iter = typename block_data::block_iterator; + + LorenzoPredictor() { this->noise = 0; } + + LorenzoPredictor(double eb) { + this->noise = 0; + if (L == 1) { + if (N == 1) { + this->noise = 0.5 * eb; + } else if (N == 2) { + this->noise = 0.81 * eb; + } else if (N == 3) { + this->noise = 1.22 * eb; + } else if (N == 4) { + this->noise = 1.79 * eb; + } + } else if (L == 2) { + if (N == 1) { + this->noise = 1.08 * eb; + } else if (N == 2) { + this->noise = 2.76 * eb; + } else if (N == 3) { + this->noise = 6.8 * eb; + } + } + } + + bool precompress(const block_iter &) override { return true; } + + void precompress_block_commit() noexcept override {} + + bool predecompress(const block_iter &) override { return true; } + + void save(uchar *&) override {} + + void load(const uchar *&, size_t &) override {} + + void print() const override { + std::cout << L << "-Layer " << N << "D Lorenzo predictor, noise = " << noise << "\n"; + } + + size_t get_padding() override { return 2; } + + T estimate_error(const block_iter &block, T *d, const std::array &index) override { + return fabs(*d - predict(block, d, index)) + this->noise; + } + + T predict(const block_iter &block, T *d, const std::array & /*index*/) override { + auto ds = block.get_dim_strides(); + if constexpr (N == 1 && L == 1) { + return prev1(d, 1); + } else if constexpr (N == 2 && L == 1) { + return prev2(d, ds, 0, 1) + prev2(d, ds, 1, 0) - prev2(d, ds, 1, 1); + } else if constexpr (N == 3 && L == 1) { + return prev3(d, ds, 0, 0, 1) + prev3(d, ds, 0, 1, 0) + prev3(d, ds, 1, 0, 0) - prev3(d, ds, 0, 1, 1) - + prev3(d, ds, 1, 0, 1) - prev3(d, ds, 1, 1, 0) + prev3(d, ds, 1, 1, 1); + } else if constexpr (N == 4 && L == 1) { + return prev4(d, ds, 0, 0, 0, 1) + prev4(d, ds, 0, 0, 1, 0) - prev4(d, ds, 0, 0, 1, 1) + + prev4(d, ds, 0, 1, 0, 0) - prev4(d, ds, 0, 1, 0, 1) - prev4(d, ds, 0, 1, 1, 0) + + prev4(d, ds, 0, 1, 1, 1) + prev4(d, ds, 1, 0, 0, 0) - prev4(d, ds, 1, 0, 0, 1) - + prev4(d, ds, 1, 0, 1, 0) + prev4(d, ds, 1, 0, 1, 1) - prev4(d, ds, 1, 1, 0, 0) + + prev4(d, ds, 1, 1, 0, 1) + prev4(d, ds, 1, 1, 1, 0) - prev4(d, ds, 1, 1, 1, 1); + } else if constexpr (N == 1 && L == 2) { + return 2 * prev1(d, 1) - prev1(d, 2); + } else if constexpr (N == 2 && L == 2) { + return 2 * prev2(d, ds, 0, 1) - prev2(d, ds, 0, 2) + 2 * prev2(d, ds, 1, 0) - 4 * prev2(d, ds, 1, 1) + + 2 * prev2(d, ds, 1, 2) - prev2(d, ds, 2, 0) + 2 * prev2(d, ds, 2, 1) - prev2(d, ds, 2, 2); + } else if constexpr (N == 3 && L == 2) { + return 2 * prev3(d, ds, 0, 0, 1) - prev3(d, ds, 0, 0, 2) + 2 * prev3(d, ds, 0, 1, 0) - + 4 * prev3(d, ds, 0, 1, 1) + 2 * prev3(d, ds, 0, 1, 2) - prev3(d, ds, 0, 2, 0) + + 2 * prev3(d, ds, 0, 2, 1) - prev3(d, ds, 0, 2, 2) + 2 * prev3(d, ds, 1, 0, 0) - + 4 * prev3(d, ds, 1, 0, 1) + 2 * prev3(d, ds, 1, 0, 2) - 4 * prev3(d, ds, 1, 1, 0) + + 8 * prev3(d, ds, 1, 1, 1) - 4 * prev3(d, ds, 1, 1, 2) + 2 * prev3(d, ds, 1, 2, 0) - + 4 * prev3(d, ds, 1, 2, 1) + 2 * prev3(d, ds, 1, 2, 2) - prev3(d, ds, 2, 0, 0) + + 2 * prev3(d, ds, 2, 0, 1) - prev3(d, ds, 2, 0, 2) + 2 * prev3(d, ds, 2, 1, 0) - + 4 * prev3(d, ds, 2, 1, 1) + 2 * prev3(d, ds, 2, 1, 2) - prev3(d, ds, 2, 2, 0) + + 2 * prev3(d, ds, 2, 2, 1) - prev3(d, ds, 2, 2, 2); + } else { + static_assert(N <= 4 && L <= 2, "Unsupported dimension or layer configuration"); + return T(0); + } + } + + protected: + T noise = 0; + + private: + // Helper functions for Lorenzo prediction. The neighbour offsets are unsigned, so `d - offset` keeps + // the step negative instead of wrapping it into an out-of-bounds pointer. + T prev1(T *d, size_t i) { return *(d - i); } + T prev2(T *d, const std::array &ds, size_t j, size_t i) { return *(d - (j * ds[0] + i)); } + T prev3(T *d, const std::array &ds, size_t k, size_t j, size_t i) { + return *(d - (k * ds[1] + j * ds[0] + i)); + } + T prev4(T *d, const std::array &ds, size_t t, size_t k, size_t j, size_t i) { + return *(d - (t * ds[2] + k * ds[1] + j * ds[0] + i)); + } +}; +} // namespace SZ3 +#endif diff --git a/include/SZ3/predictor/Predictor.hpp b/include/SZ3/predictor/Predictor.hpp new file mode 100644 index 00000000..7fa9014d --- /dev/null +++ b/include/SZ3/predictor/Predictor.hpp @@ -0,0 +1,76 @@ +#ifndef SZ3_PREDICTOR_HPP +#define SZ3_PREDICTOR_HPP + +#include "SZ3/def.hpp" +#include "SZ3/utils/BlockwiseIterator.hpp" + +namespace SZ3::concepts { + +/** + * Data prediction interface + * BlockwiseDecomposition will automatically iterate through multidimensional data to apply the + * Predictor on each data point. + * @tparam T original data type + * @tparam N original data dimension + * + */ +template +class PredictorInterface { + public: + virtual ~PredictorInterface() = default; + + using block_iter = typename block_data::block_iterator; + + /** + * predict the value for a single data point + * The index is relative to the block, not to the whole field. + * @return the predicted value + */ + virtual T predict(const block_iter &, T *, const std::array &) = 0; + + /** + * estimate the prediction error ( |prediction value - read value|) for a single data point + * @return the estimated prediction error + */ + virtual T estimate_error(const block_iter &, T *, const std::array &) = 0; + + /** + * compute auxiliary info (e.g., coefficients) for the given data block + * @return whether the predictor is suitable for the block (e.g., data with 100x1 shape is not suitable for 2D + * regression) + */ + virtual bool precompress(const block_iter &) = 0; + + /** + * store the auxiliary info (e.g., coefficients) to this class's internal storage + */ + virtual void precompress_block_commit() = 0; + + virtual bool predecompress(const block_iter &) = 0; + + /** + * serialize the predictor and store it to a buffer + * @param c One large buffer is pre-allocated, and the start location of the serialized predictor in the buffer is + * indicated by c. After saving the predictor to the buffer, this function should change c to indicate the next + * empty location in the buffer + */ + virtual void save(uchar *&c) = 0; + + /** + * deserialize the predictor from a buffer + * @param c start location of the predictor in the buffer + * @param remaining_length the remaining length of the buffer + */ + virtual void load(const uchar *&c, size_t &remaining_length) = 0; + + /// Upper bound on the space save() needs for the current input. + virtual size_t size_est() { return 0; } + + virtual size_t get_padding() { return 0; } + + virtual void print() const = 0; +}; + +} // namespace SZ3::concepts + +#endif diff --git a/include/SZ3/predictor/RegressionPredictor.hpp b/include/SZ3/predictor/RegressionPredictor.hpp new file mode 100644 index 00000000..d3498879 --- /dev/null +++ b/include/SZ3/predictor/RegressionPredictor.hpp @@ -0,0 +1,187 @@ +#ifndef SZ3_REGRESSION_PREDICTOR_HPP +#define SZ3_REGRESSION_PREDICTOR_HPP + +#include +#include +#include +#include + +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/predictor/Predictor.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" + +namespace SZ3 { + +// N-d regression predictor +template > +class RegressionPredictor : public concepts::PredictorInterface { + static_assert(std::is_floating_point::value, "RegressionPredictor only takes a floating-point T"); + + public: + using block_iter = typename block_data::block_iterator; + + static const uint8_t predictor_id = 0b00000010; + + RegressionPredictor() : quantizer_independent(0), quantizer_liner(0), prev_coeffs{0}, current_coeffs{0} {} + + RegressionPredictor(uint block_size, double eb) + : quantizer_independent(eb / (N + 1)), + quantizer_liner(eb / (N + 1) / block_size), + prev_coeffs{0}, + current_coeffs{0} {} + + bool precompress(const block_iter &block) override { + auto range = block.get_block_range(); + + std::array dims{0}; + double num_elements = 1; + for (uint i = 0; i < N; i++) { + dims[i] = range[i].second - range[i].first; + if (dims[i] <= 1) { + return false; + } + num_elements *= dims[i]; + } + + std::array sum{0}; + block_iter::foreach (block, [&](T *c, const std::array &index) { + for (uint i = 0; i < N; i++) { + sum[i] += index[i] * (*c); + } + sum[N] += *c; + }); + std::fill(current_coeffs.begin(), current_coeffs.end(), 0); + current_coeffs[N] = sum[N] / num_elements; + for (uint i = 0; i < N; i++) { + current_coeffs[i] = (2 * sum[i] / (dims[i] - 1) - sum[N]) * 6 / num_elements / (dims[i] + 1); + current_coeffs[N] -= (dims[i] - 1) * current_coeffs[i] / 2; + } + return true; + } + + void precompress_block_commit() noexcept override { + pred_and_quantize_coefficients(); + prev_coeffs = current_coeffs; + } + + bool predecompress(const block_iter &block) override { + auto range = block.get_block_range(); + for (const auto &r : range) { + if (r.second - r.first <= 1) { + return false; + } + } + pred_and_recover_coefficients(); + return true; + } + + T estimate_error(const block_iter &block, T *d, const std::array &index) override { + return fabs(*d - predict(block, d, index)); + } + + T predict(const block_iter & /*block*/, T * /*d*/, const std::array &index) override { + if constexpr (N == 1) { + return nofma(current_coeffs[0] * index[0]) + current_coeffs[1]; + } else if constexpr (N == 2) { + return nofma(current_coeffs[0] * index[0]) + nofma(current_coeffs[1] * index[1]) + current_coeffs[2]; + } else if constexpr (N == 3) { + return nofma(current_coeffs[0] * index[0]) + nofma(current_coeffs[1] * index[1]) + + nofma(current_coeffs[2] * index[2]) + current_coeffs[3]; + } else if constexpr (N == 4) { + return nofma(current_coeffs[0] * index[0]) + nofma(current_coeffs[1] * index[1]) + + nofma(current_coeffs[2] * index[2]) + nofma(current_coeffs[3] * index[3]) + current_coeffs[4]; + } else { + static_assert(N <= 4, "Unsupported dimension or layer configuration"); + return T(0); + } + } + + size_t size_est() override { + if (regression_coeff_quant_inds.empty()) { + return sizeof(uint64_t); + } + // save() builds a Huffman stream over the coefficients, and nothing else counts it. + size_t states = static_cast(std::max(quantizer_independent.get_out_range().second, + quantizer_liner.get_out_range().second)) + + 1; + return sizeof(uint64_t) + quantizer_independent.size_est() + quantizer_liner.size_est() + + Encoder::size_bound(regression_coeff_quant_inds.size(), states); + } + + void save(uchar *&c) override { + write(regression_coeff_quant_inds.size(), c); + if (!regression_coeff_quant_inds.empty()) { + quantizer_independent.save(c); + quantizer_liner.save(c); + Encoder encoder = Encoder(); + encoder.preprocess_encode( + regression_coeff_quant_inds, + std::max(quantizer_independent.get_out_range().second, quantizer_liner.get_out_range().second)); + encoder.save(c); + encoder.encode(regression_coeff_quant_inds, c); + encoder.postprocess_encode(); + } + } + + void load(const uchar *&c, size_t &remaining_length) override { + uint64_t coeff_size = 0; + read(coeff_size, c, remaining_length); + if (coeff_size > 0) { + quantizer_independent.load(c, remaining_length); + quantizer_liner.load(c, remaining_length); + Encoder encoder = Encoder(); + encoder.load(c, remaining_length); + regression_coeff_quant_inds = encoder.decode(c, coeff_size, remaining_length); + encoder.postprocess_decode(); + std::fill(current_coeffs.begin(), current_coeffs.end(), 0); + regression_coeff_index = 0; + } + } + + void print() const override { + std::cout << "Regression predictor, indendent term eb = " << quantizer_independent.get_eb() << "\n"; + std::cout << "Regression predictor, linear term eb = " << quantizer_liner.get_eb() << "\n"; + // int count = 0; + // int ind = regression_coeff_index ? regression_coeff_index : regression_coeff_quant_inds.size(); + std::cout << "Prev coeffs: "; + for (const auto &c : prev_coeffs) { + std::cout << c << " "; + } + std::cout << "\nCurrent coeffs: "; + for (const auto &c : current_coeffs) { + std::cout << c << " "; + } + std::cout << std::endl; + } + + private: + LinearQuantizer quantizer_independent, quantizer_liner; + std::vector regression_coeff_quant_inds; + size_t regression_coeff_index = 0; + std::array prev_coeffs; + std::array current_coeffs; + + void pred_and_quantize_coefficients() { + for (int i = 0; i < static_cast(N); i++) { + regression_coeff_quant_inds.push_back( + quantizer_liner.quantize_and_overwrite(current_coeffs[i], prev_coeffs[i])); + } + regression_coeff_quant_inds.push_back( + quantizer_independent.quantize_and_overwrite(current_coeffs[N], prev_coeffs[N])); + } + + void pred_and_recover_coefficients() { + if (regression_coeff_index + N + 1 > regression_coeff_quant_inds.size()) + throw std::out_of_range("SZ3: ran out of regression coefficients while decompressing"); + for (int i = 0; i < static_cast(N); i++) { + current_coeffs[i] = + quantizer_liner.recover(current_coeffs[i], regression_coeff_quant_inds[regression_coeff_index++]); + } + current_coeffs[N] = + quantizer_independent.recover(current_coeffs[N], regression_coeff_quant_inds[regression_coeff_index++]); + } +}; + +} // namespace SZ3 + +#endif diff --git a/include/SZ3/quantizer/LinearQuantizer.hpp b/include/SZ3/quantizer/LinearQuantizer.hpp new file mode 100644 index 00000000..8e7c7ba0 --- /dev/null +++ b/include/SZ3/quantizer/LinearQuantizer.hpp @@ -0,0 +1,145 @@ +#ifndef SZ3_LINEAR_QUANTIZER_HPP +#define SZ3_LINEAR_QUANTIZER_HPP + +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/quantizer/Quantizer.hpp" +#include "SZ3/utils/MemoryUtil.hpp" + +namespace SZ3 { +template +class LinearQuantizer : public concepts::QuantizerInterface { + static_assert(std::is_floating_point::value, "LinearQuantizer only takes a floating-point T"); + +public: + LinearQuantizer() + : error_bound(1), + error_bound_reciprocal(1), + radius(32768) { + } + + LinearQuantizer(double eb, int r = 32768) + : error_bound(eb), + error_bound_reciprocal(1.0 / eb), + radius(r) { + assert(eb != 0); + } + + double get_eb() const { return error_bound; } + + void set_eb(double eb) { + error_bound = eb; + error_bound_reciprocal = 1.0 / eb; + } + + std::pair get_out_range() const override { return std::make_pair(0, radius * 2); } + + ALWAYS_INLINE int quantize_and_overwrite(T& data, T pred) override { + T diff = data - pred; + // NaN data makes this product NaN and infinities push it past the int64_t range; casting either is + // undefined, so the range test is in floating point. What is not representable falls through to unpred. + double scaled = fabs(diff) * this->error_bound_reciprocal; + if (scaled < this->radius * 2 - 1) { + int64_t quant_index = static_cast(scaled) + 1; + quant_index >>= 1; + int half_index = quant_index; + int quant_index_shifted; + if (diff < 0) { + quant_index_shifted = this->radius - half_index; + } else { + quant_index_shifted = this->radius + half_index; + } + T decompressed_data = recover_pred(pred, quant_index_shifted); + double err = fabs(static_cast(decompressed_data) - data); + if (err <= this->error_bound) { + data = decompressed_data; + return quant_index_shifted; + } + } + unpred.push_back(data); + return 0; + } + + // recover the data using the quantization index + ALWAYS_INLINE T recover(T pred, int quant_index) override { + if (quant_index) { + return recover_pred(pred, quant_index); + } else { + return recover_unpred(); + } + } + + ALWAYS_INLINE T recover_pred(T pred, int quant_index) { + // quant_index comes from the stream; in int, 2 * (quant_index - radius) overflows past INT_MAX/2. + // Exact for every index a valid stream carries. + return pred + nofma(2 * (static_cast(quant_index) - this->radius) * this->error_bound); + } + + ALWAYS_INLINE T recover_unpred() { + if (index >= unpred.size()) throw std::out_of_range("SZ3: ran out of unpredictable values while decompressing"); + return unpred[index++]; + } + + ALWAYS_INLINE int force_save_unpred(T ori) override { + unpred.push_back(ori); + return 0; + } + + size_t size_est() { + return sizeof(uid) + sizeof(this->error_bound) + sizeof(this->radius) + sizeof(uint64_t) + + unpred.size() * sizeof(T); + } + + void save(unsigned char*& c) const override { + write(uid, c); + write(this->error_bound, c); + write(this->radius, c); + size_t unpred_size = unpred.size(); + write(unpred_size, c); + if (unpred_size > 0) { + write(unpred.data(), unpred.size(), c); + } + } + + void load(const unsigned char*& c, size_t& remaining_length) override { + uchar uid_read; + read(uid_read, c, remaining_length); + if (uid_read != uid) { + throw std::invalid_argument("LinearQuantizer uid mismatch"); + } + read(this->error_bound, c, remaining_length); + this->error_bound_reciprocal = 1.0 / this->error_bound; + read(this->radius, c, remaining_length); + uint64_t unpred_size = 0; + read(unpred_size, c, remaining_length); + if (unpred_size > 0) { + // resize() below is sized from the stream, so check the count against the bytes that exist first. + if (unpred_size > remaining_length / sizeof(T)) + throw std::out_of_range("SZ3: unpredictable value count exceeds the compressed buffer"); + unpred.resize(unpred_size); + read(unpred.data(), unpred_size, c, remaining_length); + } + index = 0; + } + + void print() override { + printf("[LinearQuantizer] error_bound = %.8G, radius = %d, unpred = %zu\n", error_bound, radius, unpred.size()); + } + +private: + std::vector unpred; + size_t index = 0; // used in decompression only + uchar uid = 0b10; + + double error_bound; + double error_bound_reciprocal; + int radius; // quantization interval radius +}; +} // namespace SZ3 +#endif diff --git a/include/SZ3/quantizer/Quantizer.hpp b/include/SZ3/quantizer/Quantizer.hpp new file mode 100644 index 00000000..d4852feb --- /dev/null +++ b/include/SZ3/quantizer/Quantizer.hpp @@ -0,0 +1,76 @@ +#ifndef SZ3_QUANTIZER_HPP +#define SZ3_QUANTIZER_HPP + +#include +#include +#include +#include + +namespace SZ3::concepts { + +/** + * quantize a single data point with precision loss + * E.g: float->int + * @tparam Ti original data type + * @tparam To quantized data type + */ +template +class QuantizerInterface { + public: + virtual ~QuantizerInterface() = default; + + /** + * quantize the error (error=data-pred) based on error bound, and overwrite the data with reconstructed value + * @param data single data point + * @param pred predicted value for this data point + * @return quantized error + */ + virtual To quantize_and_overwrite(Ti &data, Ti pred) = 0; + + /** + * reconstructed the data point + * @param pred predicted value for the data point + * @param quant_index quantized error + * @return reconstructed value of the data point + */ + virtual Ti recover(Ti pred, To quant_index) = 0; + + virtual To force_save_unpred(Ti ori) = 0; + + /** + ** serialize the quantizer and store it to a buffer + * @param c One large buffer is pre-allocated, and the start location of the serialized quantizer in the buffer is + *indicated by c. After saving the quantizer to the buffer, this function should change c to indicate the next empty + *location in the buffer + */ + virtual void save(uchar *&c) const = 0; + + /** + * deserialize the quantizer from a buffer + * @param c start location of the quantizer in the buffer + * @param remaining_length the remaining length of the buffer + */ + virtual void load(const uchar *&c, size_t &remaining_length) = 0; + + virtual std::pair get_out_range() const = 0; + + virtual void precompress_data() {} + + virtual void predecompress_data() {} + + /** + * this function is always executed before save() + * DO NOT reset non-temporary variables (such as unpredictable_data) in this function. + */ + virtual void postcompress_data() {} + + /** + * DO NOT reset non-temporary variables (such as unpredictable_data) in this function. + */ + virtual void postdecompress_data() {} + + virtual void print() {} +}; +} // namespace SZ3::concepts + +#endif diff --git a/include/SZ3/utils/BlockwiseIterator.hpp b/include/SZ3/utils/BlockwiseIterator.hpp new file mode 100644 index 00000000..2b73e0c1 --- /dev/null +++ b/include/SZ3/utils/BlockwiseIterator.hpp @@ -0,0 +1,342 @@ +#ifndef SZ3_BLOCKWISE_ITERATOR_HPP +#define SZ3_BLOCKWISE_ITERATOR_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" + +namespace SZ3 { + +/** + * @brief A class representing block data with N dimensions. + * + * This class provides functionality for managing multidimensional data blocks, + * including iterating over blocks, copying data in and out, and handling padding. + * + * @tparam T The type of the data elements. + * @tparam N The number of dimensions. + */ +template +class block_data : public std::enable_shared_from_this> { + public: + /** + * @brief A nested class for iterating over blocks of data. + */ + class block_iterator { + public: + /** + * @brief Constructs a block iterator. + * + * @param mddata_ A shared pointer to the block data. + * @param block_size_ The size of each block. + */ + block_iterator(std::shared_ptr &&mddata_, size_t block_size_) noexcept + : mddata(mddata_), block_size(block_size_) { + std::fill(offset.begin(), offset.end(), 0); + } + + /** + * @brief Advances the iterator to the next block. + * + * @return true if the iterator successfully moved to the next block, false otherwise. + */ + ALWAYS_INLINE bool next() { + size_t i = N - 1; + offset[i] += block_size; + while (i && (offset[i] >= mddata->dims[i])) { + offset[i] = 0; + offset[--i] += block_size; + } + return (offset[0] < mddata->dims[0]); + } + + /** + * @brief Gets the range of the current block in each dimension. + * + * @return An array of pairs representing the [start, end) indices of the block in each dimension. + */ + ALWAYS_INLINE std::array, N> get_block_range() const { + std::array, N> range; + for (uint i = 0; i < N; i++) { + range[i].first = offset[i]; + range[i].second = std::min(offset[i] + block_size, mddata->dims[i]); + } + return range; + } + + /** + * @brief Gets a pointer to the data of the current block. + * + * @param args The relative indices within the block. + * @return A pointer to the data at the specified indices. + */ + template + ALWAYS_INLINE T *get_block_data(Idx... args) const { + auto strides = get_dim_strides(); + auto idx = std::array{static_cast(std::forward(args))...}; + size_t off = 0; + for (uint i = 0; i < N; i++) { + off += (idx[i] + offset[i]) * strides[i]; + } + return mddata->dataptr() + off; + } + + /** + * @brief Gets the strides for each dimension, including padding. + * + * @return An array of strides for each dimension. + */ + ALWAYS_INLINE std::array get_dim_strides() const { return mddata->ds_padding; } + + /** + * @brief Iterates over all elements in the current block and applies a function. + * + * @tparam Func The type of the function to apply. + * @param block The block iterator. + * @param func The function to apply to each element. + */ + template + static ALWAYS_INLINE void foreach (const block_iterator &block, Func && func) { + auto range = block.get_block_range(); + if constexpr (N == 1) { + T *d = block.get_block_data(0); + for (size_t i = 0; i < range[0].second - range[0].first; i++) { + func(d++, {i}); + } + } else if constexpr (N == 2) { + for (size_t i = 0; i < range[0].second - range[0].first; i++) { + T *d = block.get_block_data(i, 0); + for (size_t j = 0; j < range[1].second - range[1].first; j++) { + func(d++, {i, j}); + } + } + } else if constexpr (N == 3) { + for (size_t i = 0; i < range[0].second - range[0].first; i++) { + for (size_t j = 0; j < range[1].second - range[1].first; j++) { + T *d = block.get_block_data(i, j, 0); + for (size_t k = 0; k < range[2].second - range[2].first; k++) { + func(d++, {i, j, k}); + } + } + } + } else if constexpr (N == 4) { + for (size_t i = 0; i < range[0].second - range[0].first; i++) { + for (size_t j = 0; j < range[1].second - range[1].first; j++) { + for (size_t k = 0; k < range[2].second - range[2].first; k++) { + T *d = block.get_block_data(i, j, k, 0); + for (size_t t = 0; t < range[3].second - range[3].first; t++) { + func(d++, {i, j, k, t}); + } + } + } + } + } else { + throw std::invalid_argument("N (dimension) should be less than 5"); + } + } + + /** + * @brief Iterates over sampled elements in the current block and applies a function. + * + * @tparam Func The type of the function to apply. + * @param block The block iterator. + * @param func The function to apply to sampled elements. + */ + template + static ALWAYS_INLINE void foreach_sampling(const block_iterator &block, Func &&func) { + size_t min_size = std::numeric_limits::max(); + for (const auto &r : block.get_block_range()) { + min_size = std::min(min_size, r.second - r.first); + } + if constexpr (N == 1) { + func(block.get_block_data(0), {0}); + func(block.get_block_data(min_size - 1), {min_size - 1}); + } else if constexpr (N <= 4) { + for (size_t i = 0; i < min_size; i++) { + size_t j = min_size - 1 - i; + if constexpr (N == 2) { + func(block.get_block_data(i, i), {i, i}); + func(block.get_block_data(i, j), {i, j}); + } else if constexpr (N == 3) { + func(block.get_block_data(i, i, i), {i, i, i}); + func(block.get_block_data(i, i, j), {i, i, j}); + func(block.get_block_data(i, j, i), {i, j, i}); + func(block.get_block_data(i, j, j), {i, j, j}); + } else if constexpr (N == 4) { + func(block.get_block_data(i, i, i, i), {i, i, i, i}); + func(block.get_block_data(i, i, i, j), {i, i, i, j}); + func(block.get_block_data(i, i, j, i), {i, i, j, i}); + func(block.get_block_data(i, i, j, j), {i, i, j, j}); + func(block.get_block_data(i, j, i, i), {i, j, i, i}); + func(block.get_block_data(i, j, i, j), {i, j, i, j}); + func(block.get_block_data(i, j, j, i), {i, j, j, i}); + func(block.get_block_data(i, j, j, j), {i, j, j, j}); + } + } + } else { + throw std::invalid_argument("N (dimension) should be less than 5"); + } + } + + friend block_data; + std::shared_ptr mddata; + + private: + std::array offset; + size_t block_size; + }; + + ~block_data() { + if (padding > 0 && !internal_buffer.empty() && data_cp_dst != nullptr) { + copy_data_with_padding(data_cp_dst, ds, data_padding, ds_padding, dims); + } + } + + block_data(T *data_, const std::vector &dims_, size_t padding_ = 0, bool data_valid = true) + : padding(padding_) { + if (dims_.size() != N) { + throw std::invalid_argument("#dims does not match!"); + } + std::copy_n(dims_.begin(), N, dims.begin()); + cal_dim_strides(); + + if (padding > 0) { + internal_buffer.resize(num_padding); + size_t offset = std::accumulate(ds_padding.begin(), ds_padding.end(), static_cast(0)); + data_padding = &internal_buffer[padding * offset]; + if (data_valid) { + copy_data_with_padding(data_padding, ds_padding, data_, ds, dims); + } else { + data_cp_dst = data_; + } + } else { + data_padding = data_; + } + } + + /** + * @brief The block's `num` values in unpadded layout. With padding this is an internal copy, + * materialized on the first call and valid until this object is destroyed. + */ + const T *values() { + if (padding == 0 || internal_buffer.empty()) { + return data_padding; + } + unpadded_buffer.resize(num); + copy_data_with_padding(unpadded_buffer.data(), ds, data_padding, ds_padding, dims); + return unpadded_buffer.data(); + } + + block_iterator block_iter(size_t block_size) { return block_iterator(this->shared_from_this(), block_size); } + + protected: + ALWAYS_INLINE T *dataptr() { return data_padding; } + + private: + ALWAYS_INLINE void cal_dim_strides() { + size_t cur_stride = 1, cur_stride_pading = 1; + for (int i = N - 1; i >= 0; i--) { + ds[i] = cur_stride; + ds_padding[i] = cur_stride_pading; + cur_stride *= dims[i]; + cur_stride_pading *= (dims[i] + padding); + } + num = cur_stride; + num_padding = cur_stride_pading; + } + + void copy_data_with_padding(T *dst, const std::array &dst_stride, const T *src, + const std::array &src_stride, const std::array &dims_) { + if (dst == nullptr || src == nullptr) { + throw std::invalid_argument("Null pointer passed to copy_data_with_padding"); + return; + } + if constexpr (N == 1) { + memcpy(&dst[0], &src[0], dims_[0] * sizeof(T)); + } else if constexpr (N == 2) { + for (size_t i = 0; i < dims_[0]; i++) { + memcpy(&dst[i * dst_stride[0]], &src[i * src_stride[0]], dims_[1] * sizeof(T)); + } + } else if constexpr (N == 3) { + for (size_t i = 0; i < dims_[0]; i++) { + for (size_t j = 0; j < dims_[1]; j++) { + memcpy(&dst[i * dst_stride[0] + j * dst_stride[1]], &src[i * src_stride[0] + j * src_stride[1]], + dims_[2] * sizeof(T)); + } + } + } else if constexpr (N == 4) { + for (size_t i = 0; i < dims_[0]; i++) { + for (size_t j = 0; j < dims_[1]; j++) { + for (size_t k = 0; k < dims_[2]; k++) { + memcpy(&dst[i * dst_stride[0] + j * dst_stride[1] + k * dst_stride[2]], + &src[i * src_stride[0] + j * src_stride[1] + k * src_stride[2]], dims_[3] * sizeof(T)); + } + } + } + } else { + throw std::invalid_argument("N (dimension) should be less than 5"); + } + } + + std::array dims; // dimension + std::array ds, ds_padding; // stride + std::vector internal_buffer; + std::vector unpadded_buffer; // materialized on demand by values() + T *data_cp_dst = nullptr; + T *data_padding; // point to either data_ or internal_buffer depending on padding + size_t padding; + size_t num, num_padding; +}; + +template +ALWAYS_INLINE void foreach (T *data, size_t offset, const std::array &begins, + const std::array &ends, const std::array &strides, + const std::array &dim_offsets, Func && func) { + if constexpr (N == 1) { + for (size_t i = begins[0]; i < ends[0]; i += strides[0]) { + T *d = data + offset + i * dim_offsets[0]; + func(d); + } + } else if constexpr (N == 2) { + for (size_t i = begins[0]; i < ends[0]; i += strides[0]) { + for (size_t j = begins[1]; j < ends[1]; j += strides[1]) { + T *d = data + offset + i * dim_offsets[0] + j * dim_offsets[1]; + func(d); + } + } + } else if constexpr (N == 3) { + for (size_t i = begins[0]; i < ends[0]; i += strides[0]) { + for (size_t j = begins[1]; j < ends[1]; j += strides[1]) { + for (size_t k = begins[2]; k < ends[2]; k += strides[2]) { + T *d = data + offset + i * dim_offsets[0] + j * dim_offsets[1] + k * dim_offsets[2]; + func(d); + } + } + } + } else if constexpr (N == 4) { + for (size_t i = begins[0]; i < ends[0]; i += strides[0]) { + for (size_t j = begins[1]; j < ends[1]; j += strides[1]) { + for (size_t k = begins[2]; k < ends[2]; k += strides[2]) { + for (size_t l = begins[3]; l < ends[3]; l += strides[3]) { + T *d = data + offset + i * dim_offsets[0] + j * dim_offsets[1] + k * dim_offsets[2] + + l * dim_offsets[3]; + func(d); + } + } + } + } + } else { + throw std::invalid_argument("N (dimension) should be less than 5"); + } +} + +} // namespace SZ3 +#endif diff --git a/include/SZ3/utils/ByteUtil.hpp b/include/SZ3/utils/ByteUtil.hpp new file mode 100644 index 00000000..38337af8 --- /dev/null +++ b/include/SZ3/utils/ByteUtil.hpp @@ -0,0 +1,340 @@ +// +// Created by Kai Zhao on 1/28/21. +// + +#ifndef SZ3_BYTEUTIL_HPP +#define SZ3_BYTEUTIL_HPP + +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" + +namespace SZ3 { + +typedef union lint16 { + unsigned short usvalue; + short svalue; + unsigned char byte[2]; +} lint16; + +typedef union lint32 { + int ivalue; + unsigned int uivalue; + unsigned char byte[4]; +} lint32; + +typedef union lint64 { + int64_t lvalue; + uint64_t ulvalue; + unsigned char byte[8]; +} lint64; + +typedef union ldouble { + double value; + uint64_t lvalue; + unsigned char byte[8]; +} ldouble; + +typedef union lfloat { + float value; + unsigned int ivalue; + unsigned char byte[4]; + uint16_t int16[2]; +} lfloat; + +inline void symTransform_4bytes(uchar data[4]) { + unsigned char tmp = data[0]; + data[0] = data[3]; + data[3] = tmp; + + tmp = data[1]; + data[1] = data[2]; + data[2] = tmp; +} + +inline int16_t bytesToInt16_bigEndian(const unsigned char *bytes) { + int16_t temp = 0; + int16_t res = 0; + + temp = bytes[0] & 0xff; + res |= temp; + + res <<= 8; + temp = bytes[1] & 0xff; + res |= temp; + + return res; +} + +inline int32_t bytesToInt32_bigEndian(const unsigned char *bytes) { + int32_t temp = 0; + int32_t res = 0; + + res <<= 8; + temp = bytes[0] & 0xff; + res |= temp; + + res <<= 8; + temp = bytes[1] & 0xff; + res |= temp; + + res <<= 8; + temp = bytes[2] & 0xff; + res |= temp; + + res <<= 8; + temp = bytes[3] & 0xff; + res |= temp; + + return res; +} + +inline int64_t bytesToInt64_bigEndian(const unsigned char *b) { + int64_t temp = 0; + int64_t res = 0; + + res <<= 8; + temp = b[0] & 0xff; + res |= temp; + + res <<= 8; + temp = b[1] & 0xff; + res |= temp; + + res <<= 8; + temp = b[2] & 0xff; + res |= temp; + + res <<= 8; + temp = b[3] & 0xff; + res |= temp; + + res <<= 8; + temp = b[4] & 0xff; + res |= temp; + + res <<= 8; + temp = b[5] & 0xff; + res |= temp; + + res <<= 8; + temp = b[6] & 0xff; + res |= temp; + + res <<= 8; + temp = b[7] & 0xff; + res |= temp; + + return res; +} + +inline void int16ToBytes_bigEndian(unsigned char *b, uint16_t num) { + b[0] = static_cast(num >> 8); + b[1] = static_cast(num); +} + +inline void int32ToBytes_bigEndian(unsigned char *b, uint32_t num) { + b[0] = static_cast(num >> 24); + b[1] = static_cast(num >> 16); + b[2] = static_cast(num >> 8); + b[3] = static_cast(num); +} + +inline void int64ToBytes_bigEndian(unsigned char *b, uint64_t num) { + b[0] = static_cast(num >> 56); + b[1] = static_cast(num >> 48); + b[2] = static_cast(num >> 40); + b[3] = static_cast(num >> 32); + b[4] = static_cast(num >> 24); + b[5] = static_cast(num >> 16); + b[6] = static_cast(num >> 8); + b[7] = static_cast(num); +} + +inline std::string floatToBinary(float f) { + lfloat u; + u.value = f; + std::string str(32, '0'); + for (int i = 0; i < 32; i++) { + str[31 - i] = (u.ivalue % 2) ? '1' : '0'; + u.ivalue >>= 1; + } + return str; +} + +template +void truncateArray(T data, size_t n, int byteLen, uchar *&binary) { + lfloat bytes; + int b; + for (size_t i = 0; i < n; i++) { + bytes.value = data[i]; + for (b = 4 - byteLen; b < 4; b++) { + *binary++ = bytes.byte[b]; + } + } +} + +template +void truncateArrayRecover(uchar *binary, size_t n, int byteLen, T *data) { + lfloat bytes; + bytes.ivalue = 0; + int b; + for (size_t i = 0; i < n; i++) { + for (b = 4 - byteLen; b < 4; b++) { + bytes.byte[b] = *binary++; + } + data[i] = bytes.value; + } +} + +template +uint8_t vector_bit_width(const std::vector &data) { + if (data.empty()) return 0; + T max_value = *std::max_element(data.begin(), data.end()); + uint8_t bits = 0; + while (max_value > 0) { + max_value >>= 1; + ++bits; + } + return bits; +} + +template +void vector2bytes(const std::vector &data, uint8_t bit_width, unsigned char *&c) { + if (data.empty()) return; + + size_t current_bit = 0; + size_t byte_index = 0; + unsigned char current_byte = 0; + + for (T value : data) { + size_t bits_remaining = bit_width; + while (bits_remaining > 0) { + size_t space_in_current_byte = 8 - (current_bit % 8); + size_t bits_to_write = std::min(bits_remaining, space_in_current_byte); + size_t bits_shift = (bit_width - bits_remaining); + unsigned char bits_to_store = (value >> bits_shift) & ((1 << bits_to_write) - 1); + + current_byte |= (bits_to_store << (current_bit % 8)); + current_bit += bits_to_write; + bits_remaining -= bits_to_write; + + if (current_bit % 8 == 0) { + c[byte_index++] = current_byte; + current_byte = 0; + } + } + } + + if (current_bit % 8 != 0) { + c[byte_index++] = current_byte; + } + + c += byte_index; +} + +template +std::vector bytes2vector(const unsigned char *&c, uint8_t bit_width, size_t num_elements) { + // uint8_t bit_width = *c++; + + std::vector data(num_elements); + + size_t total_bits = num_elements * bit_width; + size_t total_bytes = (total_bits + 7) / 8; + + for (size_t i = 0; i < num_elements; ++i) { + T value = 0; + for (uint8_t j = 0; j < bit_width; ++j) { + size_t bit_index = i * bit_width + j; + size_t byte_index = bit_index / 8; + size_t bit_offset = bit_index % 8; + + value |= static_cast((c[byte_index] >> bit_offset) & 1) << j; + } + data[i] = value; + } + + c += total_bytes; + + return data; +} + +/// Signed to unsigned with small magnitudes kept small: 0, -1, 1, -2, 2, ... to 0, 1, 2, 3, 4, ... +inline uint64_t zigzag(int64_t v) { return (uint64_t(v) << 1) ^ uint64_t(v >> 63); } +inline int64_t unzigzag(uint64_t u) { return int64_t(u >> 1) ^ -int64_t(u & 1); } + +/// LEB128: 7 bits a byte, low first, the high bit set while more follow; at most 10 bytes. +inline void write_varint(uint64_t v, uchar *&p) { + for (; v >= 128; v >>= 7) *p++ = uchar(v | 128); + *p++ = uchar(v); +} +inline uint64_t read_varint(const uchar *&p, size_t &remaining_length) { + uint64_t v = 0; + for (unsigned shift = 0;; shift += 7) { + if (remaining_length == 0 || shift > 63) throw std::out_of_range("SZ3: truncated or overlong varint"); + const uchar b = *p++; + remaining_length--; + if (shift == 63 && b > 1) throw std::out_of_range("SZ3: truncated or overlong varint"); // past 64 bits + v |= uint64_t(b & 127) << shift; + if (b < 128) return v; + } +} + +/// Bits appended LSB first to a byte vector, up to 56 at a time; the bytes are added by flush(). +class BitAppender { + public: + explicit BitAppender(std::vector &out) : out_(out) {} + void put(uint64_t v, int bits) { // v < 2^bits + pending_ |= v << pending_bits_; + pending_bits_ += bits; + if (pending_bits_ < 64) return; + words_.push_back(pending_); // whole words, turned into bytes by flush() + pending_bits_ -= 64; + pending_ = v >> (bits - pending_bits_); + } + void flush() { + words_.push_back(pending_); + const size_t size = out_.size(), bytes = (words_.size() - 1) * 8 + size_t(pending_bits_ + 7) / 8; + out_.resize(size + bytes); + for (size_t k = 0; k < bytes; k++) out_[size + k] = uchar(words_[k / 8] >> (k % 8 * 8)); + words_.clear(); + pending_ = 0; + pending_bits_ = 0; + } + + private: + std::vector &out_; + std::vector words_; + uint64_t pending_ = 0; + int pending_bits_ = 0; +}; + +/// Bits read LSB first, up to 56 at a time; reading past the end throws. +class BitConsumer { + public: + BitConsumer(const uchar *begin, const uchar *end) : p_(begin), end_(end) {} + uint64_t get(int bits) { + for (; pending_bits_ < bits; pending_bits_ += 8) { + if (p_ == end_) throw std::out_of_range("SZ3: bits read past the end"); + pending_ |= uint64_t(*p_++) << pending_bits_; + } + const uint64_t v = pending_ & ((uint64_t(1) << bits) - 1); + pending_ >>= bits; + pending_bits_ -= bits; + return v; + } + bool at_end() const { return p_ == end_; } // every byte taken + + private: + const uchar *p_, *end_; + uint64_t pending_ = 0; + int pending_bits_ = 0; +}; + +} // namespace SZ3 +#endif // SZ3_BYTEUTIL_HPP diff --git a/include/SZ3/utils/Config.hpp b/include/SZ3/utils/Config.hpp new file mode 100644 index 00000000..f560c9c2 --- /dev/null +++ b/include/SZ3/utils/Config.hpp @@ -0,0 +1,516 @@ +/** + * @file Config.hpp + * @brief This file defines the configuration class and related utilities for SZ3. + * + * @note Do not delete any existing fields for backward compatibility. + * These fields are critical for ensuring that serialized configurations + * from older versions of SZ3 remain compatible with newer versions. + * Removing or altering fields could break deserialization or cause + * unexpected behavior in applications relying on SZ3. + */ + +#ifndef SZ3_Config_HPP +#define SZ3_Config_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "SZ3/version.hpp" + +#define SZ_FLOAT 0 +#define SZ_DOUBLE 1 +#define SZ_UINT8 2 +#define SZ_INT8 3 +#define SZ_UINT16 4 +#define SZ_INT16 5 +#define SZ_UINT32 6 +#define SZ_INT32 7 +#define SZ_UINT64 8 +#define SZ_INT64 9 + +namespace SZ3 { + +/** + * @enum EB + * @brief Enumeration for error bound modes. + * + * - EB_ABS: Absolute error bound. Limits the (de)compression errors to be within an absolute error. For example, + * absErrorBound=0.0001 means the decompressed value must be in [V-0.0001,V+0.0001], where V is the original true value. + * - EB_REL: Relative error bound ratio. Limits the (de)compression errors by considering the global data value range + * size (max_value - min_value). For example, relErrorBound=0.01 means the error bound is 0.01 * (max - min). + * - EB_PSNR: Peak Signal-to-Noise Ratio error bound. Expected PSNR. Only valid when ErrorBoundMode = PSNR. + * - EB_L2NORM: L2 norm error bound. Expected L2 norm error: sqrt((x1-x1')^2 + (x2-x2')^2 + ... + (xN-xN')^2) + * - EB_ABS_AND_REL: Combined absolute and relative error bound. Both absolute and relative bounds must be satisfied. + * - EB_ABS_OR_REL: Either absolute or relative error bound. Either the absolute or relative bound must be satisfied + * (whichever is looser). + */ +enum EB { EB_ABS, EB_REL, EB_PSNR, EB_L2NORM, EB_ABS_AND_REL, EB_ABS_OR_REL }; + +/** + * @enum ALGO + * @brief Enumeration for compression algorithms. + * + * - ALGO_LORENZO_REG: SZ2 algorithm (Lorenzo with regression). + * - ALGO_INTERP_LORENZO: the default SZ3 algorithm (Interpolation with Lorenzo, with tuning). + * - ALGO_INTERP: Interpolation-only (without tuning). + * - ALGO_NOPRED: No prediction. + * - ALGO_LOSSLESS: Lossless compression. + * - ALGO_BIOMD: data compression algorithm for biology molecular data. + * - ALGO_BIOMDXTC: removed in 3.4.0, kept so that code naming it compiles; compressing with it throws. Use ALGO_BIOMD. + */ +enum ALGO { ALGO_LORENZO_REG, ALGO_INTERP_LORENZO, ALGO_INTERP, ALGO_NOPRED, ALGO_LOSSLESS, ALGO_BIOMD, ALGO_BIOMDXTC }; + +/** + * @enum INTERP_ALGO + * @brief Enumeration for interpolation algorithms. + * + * - INTERP_ALGO_LINEAR: Linear interpolation. + * - INTERP_ALGO_CUBIC: Cubic interpolation. + */ +enum INTERP_ALGO { INTERP_ALGO_LINEAR, INTERP_ALGO_CUBIC }; + +const std::map ALGO_MAP = { + {"ALGO_LORENZO_REG", ALGO_LORENZO_REG}, {"ALGO_INTERP_LORENZO", ALGO_INTERP_LORENZO}, + {"ALGO_INTERP", ALGO_INTERP}, {"ALGO_NOPRED", ALGO_NOPRED}, + {"ALGO_LOSSLESS", ALGO_LOSSLESS}, {"ALGO_BIOMD", ALGO_BIOMD}, + {"ALGO_BIOMDXTC", ALGO_BIOMDXTC}, +}; + +const std::map EB_MAP = { + {"ABS", EB_ABS}, + {"REL", EB_REL}, + {"PSNR", EB_PSNR}, + {"NORM", EB_L2NORM}, + {"ABS_AND_REL", EB_ABS_AND_REL}, + {"ABS_OR_REL", EB_ABS_OR_REL}, +}; + +const std::map INTERP_ALGO_MAP = { + {"INTERP_ALGO_LINEAR", INTERP_ALGO_LINEAR}, + {"INTERP_ALGO_CUBIC", INTERP_ALGO_CUBIC}, +}; + +ALWAYS_INLINE std::string to_lower(const std::string& s) { + std::string out = s; + std::transform(out.begin(), out.end(), out.begin(), ::tolower); + return out; +} + +template +void match_enum(const std::string& input, const std::map& table, uint8_t& out) { + std::string input_lc = to_lower(input); + for (const auto& [key, val] : table) { + if (to_lower(key) == input_lc) { + out = static_cast(val); + } + } +} + +template +std::string enum_to_string(EnumType value, const std::map& forward_map) { + for (const auto& [str, val] : forward_map) { + if (val == value) { + return str; + } + } + return ""; +} + +/** + * @class Config + * @brief Configuration class for SZ3. + * + * The Config class stores various parameters and settings used by SZ3, such as: + * - Dimensions of the data. + * - Compression algorithm and error bound settings. + * - Module-specific parameters like interpolation settings. + * + * It provides methods to load/save configurations from/to INI files, serialize/deserialize configurations, + * and estimate the size of the configuration. + */ +class Config { + public: + /** + * @brief Constructor to initialize the configuration with dimensions. + * + * @tparam Dims Variadic template for dimension arguments. + * @param args Dimensions of the data. + */ + template + Config(Dims... args) { + dims = std::vector{static_cast(std::forward(args))...}; + setDims(dims.begin(), dims.end()); + } + + /** + * @brief Set the dimensions of the data. + * + * @tparam Iter Iterator type for the dimension range. + * @param begin Iterator to the beginning of the dimensions. + * @param end Iterator to the end of the dimensions. + * @return Total number of elements in the data. + */ + template + size_t setDims(Iter begin, Iter end) { + auto dims_ = std::vector(begin, end); + dims.clear(); + for (auto dim : dims_) { + if (dim > 1) { + dims.push_back(dim); + } + } + if (dims.empty()) { + dims = {1}; + } + N = dims.size(); + num = std::accumulate(dims.begin(), dims.end(), static_cast(1), std::multiplies()); + predDim = N; + blockSize = (N == 1 ? 128 : (N == 2 ? 16 : 6)); + return num; + } + + /** + * @brief Load configuration from an INI file. + * + * @param ini_file_path Path to the INI file. + * @throws std::runtime_error If the file cannot be opened. + */ + void loadcfg(const std::string& ini_file_path) { + std::ifstream file(ini_file_path); + if (!file.is_open()) { + throw std::runtime_error("Failed to open INI file: " + ini_file_path); + } + std::ostringstream buffer; + buffer << file.rdbuf(); + load_ini(buffer.str()); + } + + /** + * @brief Load configuration from an INI content string. + * + * @param ini_content Content of the INI file as a string. + */ + void load_ini(const std::string& ini_content) { + std::istringstream ss(ini_content); + std::string line, section; + + auto trim = [](std::string& s) { + s.erase(0, s.find_first_not_of(" \t\r\n")); + s.erase(s.find_last_not_of(" \t\r\n") + 1); + }; + + auto eq = [&](const std::string& a, const std::string& b) { return to_lower(a) == to_lower(b); }; + + auto parse_bool = [&](const std::string& s) { + auto ls = to_lower(s); + return ls == "true" || ls == "1" || ls == "yes" || ls == "on"; + }; + + while (std::getline(ss, line)) { + trim(line); + if (line.empty() || line[0] == '#') continue; + if (line.front() == '[') { + section = line.substr(1, line.find(']') - 1); + continue; + } + + auto sep = line.find('='); + if (sep == std::string::npos) continue; + + std::string key = line.substr(0, sep); + std::string value = line.substr(sep + 1); + trim(key); + trim(value); + + if (eq(section, "GlobalSettings")) { + if (eq(key, "CmprAlgo")) + match_enum(value, ALGO_MAP, cmprAlgo); + else if (eq(key, "ErrorBoundMode")) + match_enum(value, EB_MAP, errorBoundMode); + else if (eq(key, "AbsErrorBound")) + absErrorBound = std::stod(value); + else if (eq(key, "RelErrorBound")) + relErrorBound = std::stod(value); + else if (eq(key, "PSNRErrorBound")) + psnrErrorBound = std::stod(value); + else if (eq(key, "L2NormErrorBound")) + l2normErrorBound = std::stod(value); + else if (eq(key, "OpenMP")) + openmp = parse_bool(value); + } else if (eq(section, "AlgoSettings")) { + if (eq(key, "Lorenzo")) + lorenzo = parse_bool(value); + else if (eq(key, "Lorenzo2ndOrder")) + lorenzo2 = parse_bool(value); + else if (eq(key, "Regression")) + regression = parse_bool(value); + else if (eq(key, "Regression2ndOrder")) + regression2 = parse_bool(value); + else if (eq(key, "InterpolationAlgo")) + match_enum(value, INTERP_ALGO_MAP, interpAlgo); + else if (eq(key, "InterpolationDirection")) + interpDirection = std::stoi(value); + else if (eq(key, "BlockSize")) + blockSize = std::stoi(value); + else if (eq(key, "QuantizationBinTotal")) + quantbinCnt = std::stoi(value); + else if (eq(key, "InterpolationAnchorStride")) + interpAnchorStride = std::stoi(value); + else if (eq(key, "InterpolationAlpha")) + interpAlpha = std::stod(value); + else if (eq(key, "InterpolationBeta")) + interpBeta = std::stod(value); + } + } + } + + /** + * @brief Save the current configuration to an INI format string. + * + * @return INI format string representing the current configuration. + */ + std::string save_ini() const { + std::ostringstream ss; + + ss << "[GlobalSettings]\n"; + ss << "CmprAlgo = " << enum_to_string(static_cast(cmprAlgo), ALGO_MAP) << "\n"; + ss << "ErrorBoundMode = " << enum_to_string(static_cast(errorBoundMode), EB_MAP) << "\n"; + ss << "AbsErrorBound = " << absErrorBound << "\n"; + ss << "RelErrorBound = " << relErrorBound << "\n"; + ss << "PSNRErrorBound = " << psnrErrorBound << "\n"; + ss << "L2NormErrorBound = " << l2normErrorBound << "\n"; + ss << "OpenMP = " << (openmp ? "true" : "false") << "\n"; + + ss << "\n[AlgoSettings]\n"; + ss << "Lorenzo = " << (lorenzo ? "true" : "false") << "\n"; + ss << "Lorenzo2ndOrder = " << (lorenzo2 ? "true" : "false") << "\n"; + ss << "Regression = " << (regression ? "true" : "false") << "\n"; + ss << "Regression2ndOrder = " << (regression2 ? "true" : "false") << "\n"; + ss << "BlockSize = " << blockSize << "\n"; + ss << "QuantizationBinTotal = " << quantbinCnt << "\n"; + ss << "InterpolationAlgo = " << enum_to_string(static_cast(interpAlgo), INTERP_ALGO_MAP) << "\n"; + ss << "InterpolationDirection = " << static_cast(interpDirection) << "\n"; + ss << "InterpolationAnchorStride = " << interpAnchorStride << "\n"; + ss << "InterpolationAlpha = " << interpAlpha << "\n"; + ss << "InterpolationBeta = " << interpBeta << "\n"; + return ss.str(); + } + + /** + * @brief Serialize the configuration to a byte array. + * @note some fields are saved/loaded in specific modules, not here. + * @param c Pointer to the byte array. + * @return Size of the serialized configuration. + */ + size_t save(unsigned char*& c) const { + auto c0 = c; + c += sizeof(uchar); // reserve space for conf size + + write(N, c); + // write(dims.data(), dims.size(), c); + auto bitWidth = vector_bit_width(dims); + write(bitWidth, c); + vector2bytes(dims, bitWidth, c); + + write(num, c); + write(cmprAlgo, c); + + write(errorBoundMode, c); + if (errorBoundMode == EB_ABS) { + write(absErrorBound, c); + } else if (errorBoundMode == EB_REL) { + write(relErrorBound, c); + } else if (errorBoundMode == EB_PSNR) { + write(psnrErrorBound, c); + } else if (errorBoundMode == EB_L2NORM) { + write(l2normErrorBound, c); + } else if (errorBoundMode == EB_ABS_OR_REL) { + write(absErrorBound, c); + write(relErrorBound, c); + } else if (errorBoundMode == EB_ABS_AND_REL) { + write(absErrorBound, c); + write(relErrorBound, c); + } else { + throw std::invalid_argument("SZ3 Config::save: unknown error bound mode"); + } + + uint8_t boolvals = (lorenzo & 1) << 7 | (lorenzo2 & 1) << 6 | (regression & 1) << 5 | (regression2 & 1) << 4 | + (openmp & 1) << 3; + write(boolvals, c); + + write(dataType, c); + write(quantbinCnt, c); + write(blockSize, c); + write(predDim, c); + + const size_t written = static_cast(c - c0); + if (written > std::numeric_limits::max()) { + throw std::length_error("SZ3 Config::save: a serialized config is at most " + + std::to_string(std::numeric_limits::max()) + " bytes, this one is " + + std::to_string(written)); + } + auto confSize = static_cast(written); + write(confSize, c0); // write conf size at reserved space + return confSize; + } + + /** + * @brief Deserialize the configuration from a byte array. + * + * @param c Pointer to the byte array. + * @param remaining_length bytes readable from `c`; decremented by what is consumed. + */ + void load(const unsigned char*& c, size_t& remaining_length) { + const unsigned char* const c0 = c; + const unsigned char* const cend = c + remaining_length; + uchar confSize = 0; + read(confSize, c, remaining_length); + // Where the optional fields at the end stop. A writer that declares more than it sends stops + // at the buffer instead. + const unsigned char* const c1 = std::min(c0 + confSize, cend); + + read(N, c, remaining_length); + // Not `N > 4`: char may be signed, and then a byte like 0xF3 arrives negative and passes. + if (static_cast(N) > 4) + throw std::out_of_range("SZ3 Config::load: invalid number of dimensions"); + uint8_t bitWidth; + read(bitWidth, c, remaining_length); + if (bitWidth > 64) throw std::out_of_range("SZ3 Config::load: invalid dimension bit width"); + const size_t dim_bytes = (static_cast(N) * bitWidth + 7) / 8; + if (dim_bytes > remaining_length) throw std::out_of_range("SZ3 Config::load: dimensions exceed the buffer"); + dims = bytes2vector(c, bitWidth, N); + remaining_length -= dim_bytes; + uint64_t num64 = 0; + read(num64, c, remaining_length); + num = num64; + // num must equal the product of the dimensions, or the predictor walks more grid positions than + // were allocated. No dimensions means no elements: the HDF5 filter loads a config from cd_values + // before it knows the dataset shape. + size_t dims_product = dims.empty() ? 0 : 1; + bool dims_ok = true; + for (size_t dim : dims) { + if (dim == 0 || dims_product > std::numeric_limits::max() / dim) { + dims_ok = false; + break; + } + dims_product *= dim; + } + if (!dims_ok || dims_product != num) + throw std::out_of_range("SZ3 Config::load: dimensions inconsistent with the element count"); + read(cmprAlgo, c, remaining_length); + + read(errorBoundMode, c, remaining_length); + if (errorBoundMode == EB_ABS) { + read(absErrorBound, c, remaining_length); + } else if (errorBoundMode == EB_REL) { + read(relErrorBound, c, remaining_length); + } else if (errorBoundMode == EB_PSNR) { + read(psnrErrorBound, c, remaining_length); + } else if (errorBoundMode == EB_L2NORM) { + read(l2normErrorBound, c, remaining_length); + } else if (errorBoundMode == EB_ABS_OR_REL || errorBoundMode == EB_ABS_AND_REL) { + read(absErrorBound, c, remaining_length); + read(relErrorBound, c, remaining_length); + } else { + // save() always writes a bound here, so an unhandled mode would shift every field below it. + throw std::invalid_argument("SZ3 Config::load: unknown error bound mode"); + } + + if (c < c1) { + uint8_t boolvals; + read(boolvals, c, remaining_length); + lorenzo = (boolvals >> 7) & 1; + lorenzo2 = (boolvals >> 6) & 1; + regression = (boolvals >> 5) & 1; + regression2 = (boolvals >> 4) & 1; + openmp = (boolvals >> 3) & 1; + } + if (c < c1) read(dataType, c, remaining_length); + if (c < c1) read(quantbinCnt, c, remaining_length); + if (c < c1) read(blockSize, c, remaining_length); + if (c < c1) read(predDim, c, remaining_length); + // Skip fields a later version appends, so whatever follows this config is read from where it starts. + if (c > c1) throw std::out_of_range("SZ3 Config::load: the config is longer than its declared size"); + remaining_length -= static_cast(c1 - c); + c = c1; + } + + /** + * @brief Print the current configuration. + */ + void print() { + // printf("===================== Begin SZ3 Configuration =====================\n"); + printf("\nsz3MagicNumber = %u\n", sz3MagicNumber); + printf("sz3DataVer = %s\n", versionStr(sz3DataVer).data()); + std::cout << "Dimensions ="; + for (auto d : dims) { + std::cout << " " << d; + } + std::cout << "\n"; + std::cout << save_ini(); + } + + /** + * @brief Estimate the size of the serialized configuration. + * + * @return Estimated size in bytes. + */ + size_t size_est() const { + std::vector buffer(sizeof(Config) + 1024); + auto buffer_pos = buffer.data(); + return save(buffer_pos); + } + + uint32_t sz3MagicNumber = SZ3_MAGIC_NUMBER; ///< SZ3 magic number for identification + uint32_t sz3DataVer = versionInt(SZ3_DATA_VER); ///< SZ3 data version + + char N = 0; ///< Number of dimensions + std::vector dims; ///< Dimensions of the data + size_t num = 0; ///< Total number of data points + + /** + * @brief Global settings for compression. + * These settings are generally applicable for all users. + */ + uint8_t cmprAlgo = ALGO_INTERP_LORENZO; ///< Compression algorithm + uint8_t errorBoundMode = EB_ABS; ///< Error bound mode + double absErrorBound = 1e-3; ///< Absolute error bound + double relErrorBound = 0.0; ///< Relative error bound + double psnrErrorBound = 0.0; ///< PSNR error bound + double l2normErrorBound = 0.0; ///< L2 norm error bound + bool openmp = false; ///< Use OpenMP for parallel processing + + /** + * @brief Algorithm-specific settings. + * @note These are for advanced users or automatic tuning. + * @note Newly added settings should be saved/loaded in specific modules, not here. + */ + int quantbinCnt = 65536; ///< Maximum number of quantization intervals + int blockSize = 0; ///< Block size for processing + uint8_t predDim = 0; ///< Prediction dimension (currently unused) + uint8_t dataType = SZ_FLOAT; ///< Data type (used in HDF5 filter) + bool lorenzo = true; ///< Enable 1st order Lorenzo + bool lorenzo2 = false; ///< Enable 2nd order Lorenzo + bool regression = true; ///< Enable 1st order regression + bool regression2 = false; ///< Enable 2nd order regression + uint8_t interpAlgo = INTERP_ALGO_CUBIC; ///< Interpolation algorithm. Saved/loaded in InterpolationDecomposition. + uint8_t interpDirection = 0; ///< Interpolation direction. Saved/loaded in InterpolationDecomposition. + int interpAnchorStride = + -1; ///< Interpolation anchor stride (-1 for dynamic). Saved/loaded in InterpolationDecomposition. + double interpAlpha = 1.25; ///< Interpolation eb tuning parameter. Saved/loaded in InterpolationDecomposition. + double interpBeta = 2.0; ///< Interpolation eb tuning parameter. Saved/loaded in InterpolationDecomposition. +}; + +} // namespace SZ3 + +#endif // SZ_CONFIG_HPP diff --git a/include/SZ3/utils/Extraction.hpp b/include/SZ3/utils/Extraction.hpp new file mode 100644 index 00000000..1ab9db5a --- /dev/null +++ b/include/SZ3/utils/Extraction.hpp @@ -0,0 +1,250 @@ +// +// Created by Kai Zhao on 4/20/20. +// + +#ifndef SZ3_EXTRACTION_HPP +#define SZ3_EXTRACTION_HPP + +#include +#include +#include +#include + +#include "SZ3/def.hpp" +#include "SZ3/utils/Timer.hpp" + +namespace SZ3 { + +template +float cal_sampling_ratio(size_t block, size_t n, size_t dmin, std::vector dims) { + size_t sample_n = 1; + for (auto dim : dims) { + sample_n *= dim / dmin * 2 * block; + } + return sample_n * 1.0 / n; +} + +template +inline typename std::enable_if>::type sampling(T *data, std::vector dims, + size_t &sample_num, + std::vector &sample_dims, + size_t &sampling_block) { + assert(dims.size() == N); + assert(sample_dims.size() == N); + Timer timer(true); + size_t num = std::accumulate(dims.begin(), dims.end(), static_cast(1), std::multiplies()); + + size_t dmin = *std::min_element(dims.begin(), dims.end()); + sampling_block = dmin; + while (cal_sampling_ratio(sampling_block, num, dmin, dims) > 0.035) { + sampling_block--; + } + if (sampling_block * 2 > dmin) { + sampling_block = dmin / 2; + } + if (sampling_block < 9) { + sample_dims = dims; + sample_num = num; + return std::vector(); + } + size_t b0 = dims[0] / dmin; + size_t b1 = dims[1] / dmin; + size_t b2 = dims[2] / dmin; + size_t b3 = dims[3] / dmin; + sample_dims[0] = b0 * 2 * sampling_block; + sample_dims[1] = b1 * 2 * sampling_block; + sample_dims[2] = b2 * 2 * sampling_block; + sample_dims[3] = b3 * 2 * sampling_block; + size_t di, dj, dk, dt; + sample_num = sample_dims[0] * sample_dims[1] * sample_dims[2] * sample_dims[3]; + std::vector sampling_data(sample_num, 0); + + for (size_t bi = 0; bi < b0; bi++) { + for (size_t bj = 0; bj < b1; bj++) { + for (size_t bk = 0; bk < b2; bk++) { + for (size_t bt = 0; bt < b3; bt++) { + for (size_t i = 0; i < 2 * sampling_block; i++) { + for (size_t j = 0; j < 2 * sampling_block; j++) { + for (size_t k = 0; k < 2 * sampling_block; k++) { + for (size_t t = 0; t < 2 * sampling_block; t++) { + di = i < sampling_block ? i + sampling_block : dmin - 3 * sampling_block + i; + dj = j < sampling_block ? j + sampling_block : dmin - 3 * sampling_block + j; + dk = k < sampling_block ? k + sampling_block : dmin - 3 * sampling_block + k; + dt = t < sampling_block ? t + sampling_block : dmin - 3 * sampling_block + t; + size_t idx = (bi * dmin + di) * dims[1] * dims[2] * dims[3] + + (bj * dmin + dj) * dims[2] * dims[3] + (bk * dmin + dk) * dims[3] + + bt * dmin + dt; + sampling_data[(bi * 2 * sampling_block + i) * sample_dims[1] * sample_dims[2] * + sample_dims[3] + + (bj * 2 * sampling_block + j) * sample_dims[1] * sample_dims[2] + + (bk * 2 * sampling_block + k) * sample_dims[2] + + bt * 2 * sampling_block + t] = data[idx]; + } + } + } + } + } + } + } + } + // auto sampling_time = timer.stop(); + // printf("Generate sampling data, block = %lu percent = %.3f%% Time = %.3f \n", sampling_block, sample_num * + // 100.0 / num, + // sampling_time); + return sampling_data; +} + +template +inline typename std::enable_if>::type sampling(T *data, std::vector dims, + size_t &sample_num, + std::vector &sample_dims, + size_t &sampling_block) { + assert(dims.size() == N); + assert(sample_dims.size() == N); + size_t num = std::accumulate(dims.begin(), dims.end(), static_cast(1), std::multiplies()); + + size_t dmin = *std::min_element(dims.begin(), dims.end()); + sampling_block = dmin; + while (cal_sampling_ratio(sampling_block, num, dmin, dims) > 0.035) { + sampling_block--; + } + if (sampling_block * 2 > dmin) { + sampling_block = dmin / 2; + } + if (sampling_block < 9) { + sample_dims = dims; + sample_num = num; + return std::vector(); + } + size_t b0 = dims[0] / dmin; + size_t b1 = dims[1] / dmin; + size_t b2 = dims[2] / dmin; + sample_dims[0] = b0 * 2 * sampling_block; + sample_dims[1] = b1 * 2 * sampling_block; + sample_dims[2] = b2 * 2 * sampling_block; + size_t di, dj, dk; + sample_num = sample_dims[0] * sample_dims[1] * sample_dims[2]; + std::vector sampling_data(sample_num, 0); + + for (size_t bi = 0; bi < b0; bi++) { + for (size_t bj = 0; bj < b1; bj++) { + for (size_t bk = 0; bk < b2; bk++) { + for (size_t i = 0; i < 2 * sampling_block; i++) { + for (size_t j = 0; j < 2 * sampling_block; j++) { + for (size_t k = 0; k < 2 * sampling_block; k++) { + di = i < sampling_block ? i + sampling_block : dmin - 3 * sampling_block + i; + dj = j < sampling_block ? j + sampling_block : dmin - 3 * sampling_block + j; + dk = k < sampling_block ? k + sampling_block : dmin - 3 * sampling_block + k; + auto d = data[(bi * dmin + di) * dims[1] * dims[2] + (bj * dmin + dj) * dims[2] + + bk * dmin + dk]; + sampling_data[(bi * 2 * sampling_block + i) * sample_dims[1] * sample_dims[2] + + (bj * 2 * sampling_block + j) * sample_dims[2] + bk * 2 * sampling_block + + k] = d; + } + } + } + } + } + } + // auto sampling_time = timer.stop(); + // printf("Generate sampling data, block = %lu percent = %.3f%% Time = %.3f \n", sampling_block, sample_num * + // 100.0 / num, + // sampling_time); + return sampling_data; +} + +template +inline typename std::enable_if>::type sampling(T *data, std::vector dims, + size_t &sample_num, + std::vector &sample_dims, + size_t &sampling_block) { + assert(dims.size() == N); + assert(sample_dims.size() == N); + Timer timer(true); + size_t num = std::accumulate(dims.begin(), dims.end(), static_cast(1), std::multiplies()); + + size_t dmin = *std::min_element(dims.begin(), dims.end()); + sampling_block = dmin; + while (cal_sampling_ratio(sampling_block, num, dmin, dims) > 0.035) { + sampling_block--; + } + if (sampling_block * 2 > dmin) { + sampling_block = dmin / 2; + } + if (sampling_block < 9) { + sample_dims = dims; + sample_num = num; + return std::vector(); + } + size_t b0 = dims[0] / dmin; + size_t b1 = dims[1] / dmin; + sample_dims[0] = b0 * 2 * sampling_block; + sample_dims[1] = b1 * 2 * sampling_block; + size_t di, dj; + sample_num = sample_dims[0] * sample_dims[1]; + std::vector sampling_data(sample_num, 0); + + for (size_t bi = 0; bi < b0; bi++) { + for (size_t bj = 0; bj < b1; bj++) { + for (size_t i = 0; i < 2 * sampling_block; i++) { + for (size_t j = 0; j < 2 * sampling_block; j++) { + di = i < sampling_block ? i + sampling_block : dmin - 3 * sampling_block + i; + dj = j < sampling_block ? j + sampling_block : dmin - 3 * sampling_block + j; + auto d = data[(bi * dmin + di) * dims[1] + bj * dmin + dj]; + sampling_data[(bi * 2 * sampling_block + i) * sample_dims[1] + bj * 2 * sampling_block + j] = d; + } + } + } + } + // auto sampling_time = timer.stop(); + // printf("Generate sampling data, block = %lu percent = %.3f%% Time = %.3f \n", sampling_block, sample_num * + // 100.0 / num, + // sampling_time); + return sampling_data; +} + +template +inline typename std::enable_if>::type sampling(T *data, std::vector dims, + size_t &sample_num, + std::vector &sample_dims, + size_t &sampling_block) { + assert(dims.size() == N); + assert(sample_dims.size() == N); + Timer timer(true); + size_t num = std::accumulate(dims.begin(), dims.end(), static_cast(1), std::multiplies()); + + size_t dmin = *std::min_element(dims.begin(), dims.end()); + sampling_block = dmin; + while (cal_sampling_ratio(sampling_block, num, dmin, dims) > 0.035) { + sampling_block--; + } + if (sampling_block * 2 > dmin) { + sampling_block = dmin / 2; + } + if (sampling_block < 9) { + sample_dims = dims; + sample_num = num; + return std::vector(); + } + size_t b0 = dims[0] / dmin; + sample_dims[0] = b0 * 2 * sampling_block; + size_t di; + sample_num = sample_dims[0]; + std::vector sampling_data(sample_num, 0); + + for (size_t bi = 0; bi < b0; bi++) { + for (size_t i = 0; i < 2 * sampling_block; i++) { + di = i < sampling_block ? i + sampling_block : dmin - 3 * sampling_block + i; + auto d = data[bi * dmin + di]; + sampling_data[bi * 2 * sampling_block + i] = d; + } + } + // auto sampling_time = timer.stop(); + // printf("Generate sampling data, block = %lu percent = %.3f%% Time = %.3f \n", sampling_block, sample_num * + // 100.0 / num, + // sampling_time); + return sampling_data; +} +} // namespace SZ3 + +#endif diff --git a/include/SZ3/utils/FileUtil.hpp b/include/SZ3/utils/FileUtil.hpp new file mode 100644 index 00000000..53740201 --- /dev/null +++ b/include/SZ3/utils/FileUtil.hpp @@ -0,0 +1,81 @@ +// +// Created by Kai Zhao on 12/9/19. +// + +#ifndef SZ3_FILE_UTIL +#define SZ3_FILE_UTIL + +#include +#include +#include +#include +#include +#include +#include + +namespace SZ3 { + +/** + * read binary file and put it to a existing memory space + */ +template +void readfile(const char *file, const size_t num, Type *data) { + std::ifstream fin(file, std::ios::binary); + if (!fin) { + std::cerr << " Error, Couldn't find the file: " << file << "\n"; + throw std::invalid_argument("Couldn't find the file"); + } + fin.seekg(0, std::ios::end); + if (fin.tellg() / sizeof(Type) != num) { + throw std::invalid_argument("File size is not equal to the input setting"); + } + fin.seekg(0, std::ios::beg); + fin.read(reinterpret_cast(data), num * sizeof(Type)); + fin.close(); +} + +/** + * read binary file and put it to a new memory space + * @param num out. Set to the element count the file's size implies; the buffer is sized from it. + */ +template +std::unique_ptr readfile(const char *file, size_t &num) { + std::ifstream fin(file, std::ios::binary); + if (!fin) { + std::cerr << " Error, Couldn't find the file: " << file << std::endl; + throw std::invalid_argument("Couldn't find the file"); + } + fin.seekg(0, std::ios::end); + num = fin.tellg() / sizeof(Type); + fin.seekg(0, std::ios::beg); + // auto data = SZ3::compat::make_unique(num_elements); + auto data = std::make_unique(num); + fin.read(reinterpret_cast(&data[0]), num * sizeof(Type)); + fin.close(); + return data; +} + +template +void writefile(const char *file, Type *data, size_t num_elements) { + std::ofstream fout(file, std::ios::binary); + fout.write(reinterpret_cast(&data[0]), num_elements * sizeof(Type)); + fout.close(); +} + +template +void writeTextFile(const char *file, Type *data, size_t num_elements) { + std::ofstream fout(file); + if (fout.is_open()) { + for (size_t i = 0; i < num_elements; i++) { + fout << data[i] << std::endl; + } + fout.close(); + } else { + std::cerr << "Error, unable to open file for output: " << file << std::endl; + throw std::invalid_argument("Couldn't open the file for output"); + } +} + +} // namespace SZ3 + +#endif diff --git a/include/SZ3/utils/Interpolators.hpp b/include/SZ3/utils/Interpolators.hpp new file mode 100644 index 00000000..d405bd57 --- /dev/null +++ b/include/SZ3/utils/Interpolators.hpp @@ -0,0 +1,114 @@ +// +// Created by Kai Zhao on 9/1/20. +// + +#ifndef SZ3_INTERPOLATORS_HPP +#define SZ3_INTERPOLATORS_HPP + +#include "SZ3/def.hpp" + +namespace SZ3 { +template +T interp_linear(T a, T b) { + return (a + b) / 2; +} + +template +T interp_linear1(T a, T b) { + return -0.5 * a + nofma(1.5 * b); +} + +template +T interp_quad_1(T a, T b, T c) { + return (nofma(3 * a) + nofma(6 * b) - c) / 8; +} + +template +T interp_quad_2(T a, T b, T c) { + return (-a + nofma(6 * b) + nofma(3 * c)) / 8; +} + +template +T interp_quad_3(T a, T b, T c) { + return (nofma(3 * a) - nofma(10 * b) + nofma(15 * c)) / 8; +} + +template +T interp_cubic(T a, T b, T c, T d) { + return (-a + nofma(9 * b) + nofma(9 * c) - d) / 16; +} + +template +T interp_cubic_natural(T a, T b, T c, T d) { + return nofma(0.575 * (b + c)) - nofma(0.075 * (a + d)); +} + +template +T lorenzo_1d(T a, T b) { + return 2 * b - a; +} + +template +T lorenzo_2d(T a, T b, T c) { + return (b + c - a); +} + +template +T lorenzo_3d(T a, T b, T c, T d, T e,T f,T g) { + return (a - b - c + d - e + f + g); +} + + + + +template +T interp_cubic_front(T a, T b, T c, T d) { + return (nofma(5 * a) + nofma(15 * b) - nofma(5 * c) + d) / 16; +} + +template +T interp_cubic_front_2(T a, T b, T c, T d) { + return (a + nofma(6 * b) - 4 * c + d) / 4; +} + +template +T interp_cubic_back_1(T a, T b, T c, T d) { + return (a - nofma(5 * b) + nofma(15 * c) + nofma(5 * d)) / 16; +} + +template +T interp_cubic_back_2(T a, T b, T c, T d) { + return (nofma(-5 * a) + nofma(21 * b) - nofma(35 * c) + nofma(35 * d)) / 16; +} + +template +T interp_cubic2(T a, T b, T c, T d) { + return (nofma(-3 * a) + nofma(23 * b) + nofma(23 * c) - nofma(3 * d)) / 40; +} + +template +ALWAYS_INLINE T interp_akima(T a, T b, T c, T d) { + T t0 = 2 * b - a - c; + T t1 = 2 * c - b - d; + T abt0 = fabs(t0); + T abt1 = fabs(t1); + if (fabs(abt0 + abt1) > 1e-9) { + return (b + c) / 2 + (nofma(t0 * abt1) + nofma(t1 * abt0)) / 8 / (abt0 + abt1); + } else { + return (b + c) / 2; + } +} + +template +ALWAYS_INLINE T interp_pchip(T a, T b, T c, T d) { + T pchip = (b + c) / 2; + if ((b - a < 0) == (c - b < 0) && fabs(c - a) > 1e-9) { + pchip += 0.25 * (b - a) * (c - b) / (c - a); + } + if ((c - b < 0) == (d - c < 0) && fabs(d - b) > 1e-9) { + pchip -= 0.25 * (c - b) * (d - c) / (d - b); + } + return pchip; +} +} // namespace SZ3 +#endif // SZ_INTERPOLATORS_HPP diff --git a/include/SZ3/utils/Iterator.hpp b/include/SZ3/utils/Iterator.hpp new file mode 100644 index 00000000..4bff21b1 --- /dev/null +++ b/include/SZ3/utils/Iterator.hpp @@ -0,0 +1,322 @@ +#ifndef SZ3_ITERATOR_HPP +#define SZ3_ITERATOR_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" + +namespace SZ3 { +// N-dimensional multi_dimensional_range +template +class multi_dimensional_range : public std::enable_shared_from_this> { + public: + class multi_dimensional_iterator { + public: + using value_type = T; + using difference_type = std::ptrdiff_t; + using reference = T &; + using const_reference = T const &; + using pointer = T *; + using iterator_category = std::bidirectional_iterator_tag; + + ~multi_dimensional_iterator() = default; + + multi_dimensional_iterator() = default; + + multi_dimensional_iterator(multi_dimensional_iterator const &) = default; + + multi_dimensional_iterator &operator=(multi_dimensional_iterator const &) = default; + + multi_dimensional_iterator(multi_dimensional_iterator &&) noexcept = default; + + multi_dimensional_iterator &operator=(multi_dimensional_iterator &&) noexcept = default; + + multi_dimensional_iterator(std::shared_ptr &&range_, + std::size_t current_offset_) noexcept + : range(range_), local_index{}, global_offset(current_offset_) {} + + multi_dimensional_iterator &operator--() { + size_t i = N - 1; + local_index[i]--; + ptrdiff_t offset = -range->global_dim_strides[i]; + while (i && (local_index[i] < 0)) { + offset += range->dimensions[i] * range->global_dim_strides[i]; + local_index[i--] = range->dimensions[i] - 1; + offset -= range->global_dim_strides[i]; + local_index[i]--; + } + global_offset += offset; + return *this; + } + + multi_dimensional_iterator operator--(int) { + auto cpy = *this; + --(*this); + return cpy; + } + + inline multi_dimensional_iterator &operator++() { + size_t i = N - 1; + local_index[i]++; + ptrdiff_t offset = range->global_dim_strides[i]; + while (i && (local_index[i] == range->dimensions[i])) { + offset -= range->dimensions[i] * range->global_dim_strides[i]; + local_index[i--] = 0; + offset += range->global_dim_strides[i]; + local_index[i]++; + } + global_offset += offset; + return *this; + } + + multi_dimensional_iterator operator++(int) { + auto cpy = *this; + ++(*this); + return cpy; + } + + pointer operator->() { return range->data[global_offset]; } + + pointer operator->() const { return range->data[global_offset]; } + + reference operator*() { return range->data[global_offset]; } + + const_reference operator*() const { return range->data[global_offset]; } + + bool operator==(multi_dimensional_iterator const &rhs) const { return global_offset == rhs.global_offset; } + + bool operator!=(multi_dimensional_iterator const &rhs) const { return global_offset != rhs.global_offset; } + + std::array get_global_index() const { + auto offset = global_offset; + std::array global_idx{0}; + for (int i = N - 1; i >= 0; i--) { + global_idx[i] = offset % range->global_dimensions[i]; + offset /= range->global_dimensions[i]; + } + return global_idx; + } + + std::array get_local_index() const { return local_index; } + + size_t get_local_index(size_t i) const { return local_index[i]; } + + ptrdiff_t get_offset() const { return global_offset; } + + std::array get_dimensions() const { return range->get_dimensions(); } + + // assuming the iterator is at [i0, j0, k0, ...] + // return the value of [i0 - pos[0], j0 - pos[1], k0 - pos[2] ...] + // return 0 if range is exceeded + // [input] offset for all the dimensions + // [output] value of data at the target position + template + inline T prev(Args &&...pos) const { + // TODO: check int type + // TODO: change to offset map for efficiency + static_assert(sizeof...(Args) == N, "Must have the same number of arguments"); + auto offset = global_offset; + std::array args{std::forward(pos)...}; + for (int i = 0; i < N; i++) { + if (local_index[i] < args[i] && range->is_left_boundary(i)) return 0; + offset -= args[i] ? args[i] * range->global_dim_strides[i] : 0; + } + return range->data[offset]; + } + + // No support for carry set. + // For example, iterator in position (4,4) and dimension is 6x6, move(1,1) is supported but move (2,0) is not + // supported. + template + multi_dimensional_iterator &move(Args &&...pos) { + static_assert(sizeof...(Args) == N, "Must have the same number of arguments"); + std::array args{std::forward(pos)...}; + return move2(args); + } + + multi_dimensional_iterator &move2(std::array args) { + for (int i = N - 1; i >= 0; i--) { + if (args[i]) { + assert(0 <= local_index[i] + args[i]); + assert(local_index[i] + args[i] < range->dimensions[i]); + local_index[i] += args[i]; + global_offset += args[i] * range->global_dim_strides[i]; + } + } + return *this; + } + + // No support for carry set. + template + multi_dimensional_iterator &move() { + if (local_index[N - 1] < range->dimensions[N - 1] - 1) { + local_index[N - 1]++; + global_offset += range->global_dim_strides[N - 1]; + } + return *this; + } + + void print() { + std::cout << "("; + for (auto const &i : local_index) { + std::cout << i << ","; + } + std::cout << "),["; + for (auto const &i : range->dimensions) { + std::cout << i << ","; + } + std::cout << "]" << std::endl; + } + + private: + friend multi_dimensional_range; + std::shared_ptr range; + std::array local_index; // index of current_offset position + ptrdiff_t global_offset; + }; + + using iterator = multi_dimensional_iterator; + using const_iterator = multi_dimensional_iterator; + using value_type = T; + using reference = T &; + using pointer = T *; + + template + multi_dimensional_range(T *data_, ForwardIt1 global_dims_begin, ForwardIt1 global_dims_end, size_t stride_, + ptrdiff_t offset_) + : left_boundary{false}, data(data_) { + static_assert(std::is_convertible::value_type, std::size_t>::value, + "ForwardIt1 must be convertible to std::size_t"); + if (global_dims_end - global_dims_begin != N) { + std::stringstream ss; + ss << "#dimensions does not match: got " << global_dims_end - global_dims_begin << ", expected " << N; + throw std::invalid_argument(ss.str()); + } + set_access_stride(stride_); + // set global dimensions + int i = 0; + for (auto iter = global_dims_begin; iter != global_dims_end; ++iter) { + global_dimensions[i++] = *iter; + } + // size_t cur_stride = stride_; + // for (int i = N - 1; i >= 0; i--) { + // global_dim_strides[i] = cur_stride; + // cur_stride *= global_dimensions[i]; + // } + // set_dimensions(dims_begin, dims_end); + set_dimensions_auto(); + set_global_dim_strides(); + set_offsets(offset_); + } + + multi_dimensional_range(T *data_, std::array global_dims_, std::array stride_, + ptrdiff_t offset_) + : data(data_), left_boundary{false} { + set_access_stride(stride_); + // set global dimensions + global_dimensions = global_dims_; + set_dimensions_auto(); + set_global_dim_strides(); + set_offsets(offset_); + } + + multi_dimensional_iterator begin() { return multi_dimensional_iterator(this->shared_from_this(), start_offset); } + + multi_dimensional_iterator end() { return multi_dimensional_iterator(this->shared_from_this(), end_offset); } + + template + void set_dimensions(ForwardIt1 begin, ForwardIt1 end) { + int i = 0; + for (auto iter = begin; iter != end; ++iter) { + dimensions[i++] = *iter; + // std::cout << dimensions[i-1] << " "; + } + } + + void set_dimensions_auto() { + // std::cout << "dimensions: "; + for (size_t i = 0; i < dimensions.size(); i++) { + // std::cout << "g[i]=" << global_dimensions[i] << ",str=" << access_stride << " "; + dimensions[i] = (global_dimensions[i] - 1) / access_stride[i] + 1; + // std::cout << dimensions[i] << " "; + } + // std::cout << std::endl; + } + + void set_global_dim_strides() { + // std::cout << "strides: "; + size_t cur_stride = 1; + for (int i = N - 1; i >= 0; i--) { + global_dim_strides[i] = cur_stride * access_stride[i]; + cur_stride *= global_dimensions[i]; + // std::cout << dim_strides[i] << " "; + } + // std::cout << std::endl; + } + + void set_offsets(ptrdiff_t offset_) { + start_offset = offset_; + end_offset = start_offset + dimensions[0] * global_dim_strides[0]; + } + + void set_access_stride(size_t stride_) { access_stride.fill(stride_); } + + void set_access_stride(std::array stride_) { access_stride = stride_; } + + void update_block_range(multi_dimensional_iterator block, size_t block_size) { + std::array dims; + for (int i = 0; i < N; i++) { + // check boundary condition + if (block.local_index[i] == block.range->dimensions[i] - 1) { + dims[i] = global_dimensions[i] - block.local_index[i] * block.range->access_stride[i]; + } else { + dims[i] = block_size; + } + } + update_block_range(block, dims); + } + + void update_block_range(multi_dimensional_iterator block, const std::array &dims) { + dimensions = dims; + for (int i = 0; i < N; i++) { + left_boundary[i] = (block.get_local_index(i) == 0); + } + this->set_offsets(block.get_offset()); + } + + size_t get_dimensions(size_t i) const { return dimensions[i]; } + + std::array get_dimensions() const { return dimensions; } + + std::array get_global_dimensions() const { return global_dimensions; } + + bool is_left_boundary(size_t i) const { return left_boundary[i]; } + + T *get_data() { return data; } + + private: + std::array global_dimensions; + std::array global_dim_strides; + std::array dimensions; // the dimensions + std::array + left_boundary; // if current block is the left boundary, iterators go outside the boundary will return 0. + std::array access_stride; // stride for access pattern + ptrdiff_t start_offset; // offset for start point + ptrdiff_t end_offset; // offset for end point + T *data; // data pointer +}; + +} // namespace SZ3 +#endif diff --git a/include/SZ3/utils/KmeansUtil.hpp b/include/SZ3/utils/KmeansUtil.hpp new file mode 100644 index 00000000..1cfe9c82 --- /dev/null +++ b/include/SZ3/utils/KmeansUtil.hpp @@ -0,0 +1,368 @@ +// +// Created by Kai Zhao on 12/9/19. +// + +#ifndef SZ3_KMEANS_UTIL +#define SZ3_KMEANS_UTIL + +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/def.hpp" +#include "Timer.hpp" + +namespace SZ3 { + +/* + * Internal implementation of the SMAWK algorithm. + */ +template +void _smawk(const std::vector &rows, const std::vector &cols, + const std::function &lookup, std::vector *result) { + // Recursion base case + if (rows.size() == 0) return; + + // ******************************** + // * REDUCE + // ******************************** + + std::vector _cols; // Stack of surviving columns + for (size_t col : cols) { + while (true) { + if (_cols.size() == 0) break; + size_t row = rows[_cols.size() - 1]; + if (lookup(row, col) >= lookup(row, _cols.back())) break; + _cols.pop_back(); + } + if (_cols.size() < rows.size()) _cols.push_back(col); + } + + // Call recursively on odd-indexed rows + std::vector odd_rows; + for (size_t i = 1; i < rows.size(); i += 2) { + odd_rows.push_back(rows[i]); + } + _smawk(odd_rows, _cols, lookup, result); + + std::unordered_map col_idx_lookup; + for (size_t idx = 0; idx < _cols.size(); ++idx) { + col_idx_lookup[_cols[idx]] = idx; + } + + // ******************************** + // * INTERPOLATE + // ******************************** + + // Fill-in even-indexed rows + size_t start = 0; + for (size_t r = 0; r < rows.size(); r += 2) { + size_t row = rows[r]; + size_t stop = _cols.size() - 1; + if (r < rows.size() - 1) stop = col_idx_lookup[(*result)[rows[r + 1]]]; + size_t argmin = _cols[start]; + T min = lookup(row, argmin); + for (size_t c = start + 1; c <= stop; ++c) { + T value = lookup(row, _cols[c]); + if (c == start || value < min) { + argmin = _cols[c]; + min = value; + } + } + (*result)[row] = argmin; + start = stop; + } +} + +/* + * Interface for the SMAWK algorithm, for finding the minimum value in each row + * of an implicitly-defined totally monotone matrix. + */ +template +std::vector smawk(const size_t num_rows, const size_t num_cols, + const std::function &lookup) { + std::vector result; + result.resize(num_rows); + std::vector rows(num_rows); + iota(begin(rows), end(rows), 0); + std::vector cols(num_cols); + iota(begin(cols), end(cols), 0); + _smawk(rows, cols, lookup, &result); + return result; +} + +/* + * Calculates cluster costs in O(1) using prefix sum arrays. + */ +template +class CostCalculator { + std::vector cumsum; + std::vector cumsum2; + + public: + CostCalculator(const std::vector
&vec, size_t n) { + cumsum.push_back(0.0); + cumsum2.push_back(0.0); + for (size_t i = 0; i < n; ++i) { + double x = vec[i]; + cumsum.push_back(x + cumsum[i]); + cumsum2.push_back(x * x + cumsum2[i]); + } + } + + double calc(size_t i, size_t j) { + if (j < i) return 0.0; + double mu = (cumsum[j + 1] - cumsum[i]) / (j - i + 1); + double result = cumsum2[j + 1] - cumsum2[i]; + result += (j - i + 1) * (mu * mu); + result -= (2 * mu) * (cumsum[j + 1] - cumsum[i]); + return result; + } +}; + +template +class Matrix { + std::vector data; + size_t num_rows; + size_t num_cols; + + public: + Matrix(size_t num_rows, size_t num_cols) { + this->num_rows = num_rows; + this->num_cols = num_cols; + data.resize(num_rows * num_cols); + } + + inline T get(size_t i, size_t j) { return data[i * num_cols + j]; } + + inline void set(size_t i, size_t j, T value) { data[i * num_cols + j] = value; } +}; + +template +void cluster(DT *array, size_t n, int &k, size_t *clusters, DT *centroids) { + // *************************************************** + // * Sort input array and save info for de-sorting + // *************************************************** + + std::vector sort_idxs(n); + iota(sort_idxs.begin(), sort_idxs.end(), 0); + sort(sort_idxs.begin(), sort_idxs.end(), [&array](size_t a, size_t b) { return array[a] < array[b]; }); + // vector undo_sort_lookup(n); + std::vector
sorted_array(n); + for (size_t i = 0; i < n; ++i) { + sorted_array[i] = array[sort_idxs[i]]; + // undo_sort_lookup[sort_idxs[i]] = i; + } + + // *************************************************** + // * Set D and T using dynamic programming algorithm + // *************************************************** + + // Algorithm as presented in section 2.2 of (Gronlund et al., 2017). + + CostCalculator
cost_calculator(sorted_array, n); + Matrix
D(k, n); + Matrix T(k, n); + + for (size_t i = 0; i < n; ++i) { + D.set(0, i, cost_calculator.calc(0, i)); + T.set(0, i, 0); + } + + double ratio_avg = 0; + bool findk = false; + size_t bestk = 0; + for (int k_ = 1; k_ < k; ++k_) { + auto C = [&D, &k_, &cost_calculator](size_t i, size_t j) -> DT { + size_t col = i < j - 1 ? i : j - 1; + return D.get(k_ - 1, col) + cost_calculator.calc(j, i); + }; + std::vector row_argmins = smawk
(n, n, C); + for (size_t i = 0; i < row_argmins.size(); ++i) { + size_t argmin = row_argmins[i]; + DT min = C(i, argmin); + D.set(k_, i, min); + T.set(k_, i, argmin); + } + float ratio = D.get(k_ - 1, n - 1) / D.get(k_, n - 1); + ratio_avg = (ratio_avg * (k_ - 1) + ratio) / (k_); + // std::cout << k_ + 1 << " , " << D.get(k_, n - 1) << " , " << ratio << " , " << ratio_avg << std::endl; + if (ratio / ratio_avg > 1.5) { + bestk = k_ + 1; + findk = true; + } else { + if (findk) { + break; + } + } + } + + if (!findk) { + return; + } + k = bestk; + // std::cout << "# groups = " << k << std::endl; + + // *************************************************** + // * Extract cluster assignments by backtracking + // *************************************************** + + // Note: This step requires O(kn) memory usage due to saving the entire + // T matrix. However, it can be modified so that the memory usage is O(n). + // D and T would not need to be retained in full (D already doesn't need + // to be fully retained, although it currently is). + // Details are in section 3 of (Grønlund et al., 2017). + + // vector
sorted_clusters(n); + + size_t t = n; + size_t k_ = k - 1; + size_t n_ = n - 1; + // The do/while loop was used in place of: + // for (k_ = k - 1; k_ >= 0; --k_) + // to avoid wraparound of an unsigned type. + do { + size_t t_ = t; + t = T.get(k_, n_); + DT centroid = 0.0; + for (size_t i = t; i < t_; ++i) { + // sorted_clusters[i] = k_; + centroid += (sorted_array[i] - centroid) / (i - t + 1); + } + centroids[k_] = centroid; + k_ -= 1; + n_ = t - 1; + } while (t > 0); + + // *************************************************** + // * Order cluster assignments to match de-sorted + // * ordering + // *************************************************** + + // for (size_t i = 0; i < n; ++i) { + // clusters[i] = sorted_clusters[undo_sort_lookup[i]]; + // } +} + +template +int f(T data, double start_position, double offset) { + return round((data - start_position) / offset); +} + +template +int f1(T data, T *boundary, int n, double start_position, double offset) { + return round((data - start_position) / offset); +} + +template +int f2(T data, T *boundary, int n, double start_position, double offset) { + int low = 0, high = n; // numElems is the size of the array i.e arr.size() + while (low != high) { + int mid = (low + high) / 2; // Or a fancy way to avoid int overflow + if (boundary[mid] <= data) { + low = mid + 1; + } else { + high = mid; + } + } + return high; +} + +template +int f3(T data, T *boundary, int n, double start_position, double offset) { + for (int i = 1; i < n; i++) { + if (boundary[i] > data) { + return i - 1; + } + } + return n - 1; +} + +template +void get_cluster(T *data, size_t num, float &level_start, float &level_offset, int &level_num, size_t sample_num) { + T max = *std::max_element(data, data + num); + // T min = *std::min_element(data, data + num); + SZ3::Timer timer; + timer.start(); + std::vector sample; + if (num == sample_num) { + sample = std::vector(data, data + num); + } else { + sample.resize(sample_num); // the loop below writes through operator[] + std::random_device rd; // Will be used to obtain a seed for the random number engine + std::mt19937 gen(rd()); // Standard mersenne_twister_engine seeded with rd() + // std::uniform_int_distribution<> dis(0, 2 * sample_rate); + size_t input_idx = 0; + // for (int i = 0; i < num / sample_rate; i++) { + // input_idx = (input_idx + dis(gen)) % num; + // std::cout << input_idx << " "; + // sample[i] = input[input_idx]; + // } + // std::cout << std::endl; + std::uniform_int_distribution<> dis2(0, static_cast(num) - 1); + std::unordered_set sampledkeys; + // printf("total_num=%lu, sample_num=%lu\n", num, sample_num); + for (size_t i = 0; i < sample_num; i++) { + do { + input_idx = dis2(gen); + } while (sampledkeys.find(input_idx) != sampledkeys.end()); + // std::cout << input_idx << " "; + sample[i] = data[input_idx]; + } + // std::cout << std::endl; + + // std::sample(input.begin(), input.end(), + // sample.begin(), + // num / sample_rate, + // std::mt19937{std::random_device{}()}); + } + + timer.stop("random sample"); + // sample = input; + std::vector idx(num); + + timer.start(); + int k = 150; + std::vector cents(k); + cluster(sample.data(), sample_num, k, idx.data(), cents.data()); + // cluster(input.get(), num, 16, idx.data(), cents.data()); + timer.stop("kmeans1d"); + if (k == 150) { + // std::cout << "No clusters are found." << std::endl; + level_num = 0; + return; + } + + // std::cout << "centers : "; + // for (size_t i = 0; i < k; i++) { + // std::cout << cents[i] << " "; + // } + // std::cout << std::endl; + + // std::cout << "center diff : "; + // for (size_t i = 1; i < k; i++) { + // std::cout << cents[i] - cents[i - 1] << " "; + // } + // std::cout << std::endl; + + // std::vector boundary(k); + // boundary[0] = std::numeric_limits::min(); + // for (size_t i = 1; i < k; i++) { + // boundary[i] = (cents[i - 1] + cents[i]) / 2; + // } + + level_offset = (cents[k - 1] - cents[0]) / (k - 1); + level_start = cents[0]; + for (int i = 1; i < k; i++) { + level_start += cents[i] - i * level_offset; + } + level_start /= k; + level_num = f(max, level_start, level_offset) + 1; +} +} // namespace SZ3 + +#endif diff --git a/include/SZ3/utils/MemoryUtil.hpp b/include/SZ3/utils/MemoryUtil.hpp new file mode 100644 index 00000000..6a198302 --- /dev/null +++ b/include/SZ3/utils/MemoryUtil.hpp @@ -0,0 +1,149 @@ +// +// Created by Kai Zhao on 4/21/20. +// + +#ifndef SZ3_MEMORYOPS_HPP +#define SZ3_MEMORYOPS_HPP + +#include +#include +#include + +#include "SZ3/def.hpp" + +namespace SZ3 { + +// Endianness detection: SZ3 always stores data in little-endian format +// On big-endian systems, byte-swapping is performed during read/write +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + #define SZ3_BIG_ENDIAN 1 +#elif defined(__BIG_ENDIAN__) || defined(__ARMEB__) || defined(__THUMBEB__) || \ + defined(__AARCH64EB__) || defined(_MIPSEB) || defined(__MIPSEB) + #define SZ3_BIG_ENDIAN 1 +#else + // Default: little-endian + #define SZ3_BIG_ENDIAN 0 +#endif + +#if SZ3_BIG_ENDIAN +// Byte-swap intrinsics for fast endianness conversion +#if defined(_MSC_VER) + #include // Use for _byteswap_... + #define SZ3_HAS_BUILTIN_BSWAP 1 + #define BSWAP16(x) _byteswap_ushort(x) + #define BSWAP32(x) _byteswap_ulong(x) + #define BSWAP64(x) _byteswap_uint64(x) +#elif defined(__GNUC__) || defined(__clang__) + #define SZ3_HAS_BUILTIN_BSWAP 1 + #define BSWAP16(x) __builtin_bswap16(x) + #define BSWAP32(x) __builtin_bswap32(x) + #define BSWAP64(x) __builtin_bswap64(x) +#else + #define SZ3_HAS_BUILTIN_BSWAP 0 +#endif + +// Generic byteswap dispatcher +template +inline T byteswap(T value) { + union { T val; uint8_t bytes[sizeof(T)]; uint16_t u16; uint32_t u32; uint64_t u64; } u; + u.val = value; + + if constexpr (sizeof(T) == 1) { + return value; +#if SZ3_HAS_BUILTIN_BSWAP + } else if constexpr (sizeof(T) == 2) { + u.u16 = BSWAP16(u.u16); + } else if constexpr (sizeof(T) == 4) { + u.u32 = BSWAP32(u.u32); + } else if constexpr (sizeof(T) == 8) { + u.u64 = BSWAP64(u.u64); +#endif + } else { + // Fallback for all sizes when no intrinsics, or for sizes > 8 bytes + for (size_t i = 0; i < sizeof(T) / 2; i++) { + uint8_t tmp = u.bytes[i]; + u.bytes[i] = u.bytes[sizeof(T) - 1 - i]; + u.bytes[sizeof(T) - 1 - i] = tmp; + } + } + return u.val; +} +#endif // SZ3_BIG_ENDIAN + +// read array +template +void read(T1 *array, size_t num_elements, uchar const *&compressed_data_pos, size_t &remaining_length) { + if (num_elements > remaining_length / sizeof(T1)) + throw std::out_of_range("SZ3: attempt to read past the end of the compressed buffer"); + memcpy(array, compressed_data_pos, num_elements * sizeof(T1)); + if constexpr (SZ3_BIG_ENDIAN) { + for (size_t i = 0; i < num_elements; i++) { + array[i] = byteswap(array[i]); + } + } + remaining_length -= num_elements * sizeof(T1); + compressed_data_pos += num_elements * sizeof(T1); +} + +// read array +template +void read(T1 *array, size_t num_elements, uchar const *&compressed_data_pos) { + memcpy(array, compressed_data_pos, num_elements * sizeof(T1)); + if constexpr (SZ3_BIG_ENDIAN) { + for (size_t i = 0; i < num_elements; i++) { + array[i] = byteswap(array[i]); + } + } + compressed_data_pos += num_elements * sizeof(T1); +} + +// read variable +template +void read(T1 &var, uchar const *&compressed_data_pos) { + memcpy(&var, compressed_data_pos, sizeof(T1)); + if constexpr (SZ3_BIG_ENDIAN) { + var = byteswap(var); + } + compressed_data_pos += sizeof(T1); +} + +// read variable +template +void read(T1 &var, uchar const *&compressed_data_pos, size_t &remaining_length) { + if (sizeof(T1) > remaining_length) + throw std::out_of_range("SZ3: attempt to read past the end of the compressed buffer"); + memcpy(&var, compressed_data_pos, sizeof(T1)); + if constexpr (SZ3_BIG_ENDIAN) { + var = byteswap(var); + } + remaining_length -= sizeof(T1); + compressed_data_pos += sizeof(T1); +} + +// write array +template +void write(T1 const *array, size_t num_elements, uchar *&compressed_data_pos) { + memcpy(compressed_data_pos, array, num_elements * sizeof(T1)); + if constexpr (SZ3_BIG_ENDIAN) { + T1 *dest = reinterpret_cast(compressed_data_pos); + for (size_t i = 0; i < num_elements; i++) { + dest[i] = byteswap(dest[i]); + } + } + compressed_data_pos += num_elements * sizeof(T1); +} + +// write variable +template +void write(T1 const var, uchar *&compressed_data_pos) { + if constexpr (SZ3_BIG_ENDIAN) { + T1 le_var = byteswap(var); + memcpy(compressed_data_pos, &le_var, sizeof(T1)); + } else { + memcpy(compressed_data_pos, &var, sizeof(T1)); + } + compressed_data_pos += sizeof(T1); +} + +} // namespace SZ3 +#endif // SZ_MEMORYOPS_HPP diff --git a/include/SZ3/utils/QuantOptimization.hpp b/include/SZ3/utils/QuantOptimization.hpp new file mode 100644 index 00000000..9ae0a3a3 --- /dev/null +++ b/include/SZ3/utils/QuantOptimization.hpp @@ -0,0 +1,158 @@ +#ifndef SZ3_optimize_quant_intervals_hpp +#define SZ3_optimize_quant_intervals_hpp + +#include +#include + +#include "SZ3/def.hpp" + +namespace SZ3 { + +#define QuantIntvMeanCapacity 8192 +#define QuantIntvSampleDistance 100 +#define QuantIntvSampleCapacity 32768 +#define QuantIntvAccThreshold 0.999 + +// copied from conf.c +unsigned int static round_up_power_of_2(unsigned int base) { + base -= 1; + base = base | (base >> 1); + base = base | (base >> 2); + base = base | (base >> 4); + base = base | (base >> 8); + base = base | (base >> 16); + return base + 1; +} + +unsigned int static inline estimate_quantization_intervals(const std::vector &intervals, size_t sample_count) { + size_t target = sample_count * QuantIntvAccThreshold; + size_t sum = 0; + size_t i = 0; + for (i = 0; i < intervals.size(); i++) { + sum += intervals[i]; + if (sum > target) break; + } + if (i == intervals.size()) i = intervals.size() - 1; + unsigned int accIntervals = 2 * (i + 1); + unsigned int num_intervals = 2 * round_up_power_of_2(accIntervals); + return (num_intervals > 32) ? num_intervals : 32; +} + +template +float estimate_mean_freq_and_position(const std::vector &freq_intervals, double precision, size_t sample_count, + T &mean_guess) { + size_t max_sum = 0; + size_t max_index = 0; + size_t tmp_sum = 0; + for (size_t i = 1; i < freq_intervals.size() - 2; i++) { + tmp_sum = freq_intervals[i] + freq_intervals[i + 1]; + if (tmp_sum > max_sum) { + max_sum = tmp_sum; + max_index = i; + } + } + mean_guess += precision * static_cast(max_index + 1 - (freq_intervals.size() >> 1)); + return max_sum * 1.0 / sample_count; +} + +template +float sample_rough_mean_3d(const T *data, size_t r1, size_t r2, size_t r3, size_t sample_distance) { + double mean = 0; + size_t len = r1 * r2 * r3; + const T *data_pos = data; + size_t offset_count = 0; + size_t offset_count_2 = 0; + size_t mean_count = 0; + while (data_pos - data < len) { + mean += *data_pos; + mean_count++; + data_pos += sample_distance; + offset_count += sample_distance; + offset_count_2 += sample_distance; + if (offset_count >= r3) { + offset_count = 0; + data_pos -= 1; + } + if (offset_count_2 >= r2 * r3) { + offset_count_2 = 0; + data_pos -= 1; + } + } + if (mean_count > 0) mean /= mean_count; + return mean; +} + +template +ALWAYS_INLINE T lorenzo_predict_3d(const T *data_pos, size_t dim0_offset, size_t dim1_offset) { + return *(data_pos - 1) + *(data_pos - dim1_offset) + *(data_pos - dim0_offset) - *(data_pos - (dim1_offset + 1)) - + *(data_pos - (dim0_offset + 1)) - *(data_pos - (dim0_offset + dim1_offset)) + + *(data_pos - (dim0_offset + dim1_offset + 1)); +} + +template +int optimize_quant_invl_3d(const T *data, size_t r1, size_t r2, size_t r3, double precision, float &pred_freq, + float &mean_freq, T &mean_guess) { + float mean_rough = sample_rough_mean_3d(data, r1, r2, r3, sqrt(r1 * r2 * r3)); + std::vector intervals = std::vector(QuantIntvSampleCapacity, 0); + std::vector freq_intervals = std::vector(QuantIntvMeanCapacity, 0); + size_t freq_count = 0; + size_t sample_count = 0; + size_t sample_distance = QuantIntvSampleDistance; + size_t offset_count = sample_distance - 2; // count r3 offset + size_t offset_count_2 = 0; + size_t r23 = r2 * r3; + size_t len = r1 * r23; + const T *data_pos = data + r23 + r3 + offset_count; + size_t n1_count = 1, n2_count = 1; // count i,j sum + T pred_value = 0; + double mean_diff = 0; + ptrdiff_t freq_index = 0; + size_t pred_index = 0; + float pred_err = 0; + int radius = (QuantIntvMeanCapacity >> 1); + while (data_pos - data < len) { + pred_value = lorenzo_predict_3d(data_pos, r23, r3); + pred_err = fabs(pred_value - *data_pos); + if (pred_err < precision) freq_count++; + pred_index = (pred_err / precision + 1) / 2; + if (pred_index >= intervals.size()) { + pred_index = intervals.size() - 1; + } + intervals[pred_index]++; + + mean_diff = *data_pos - mean_rough; + if (mean_diff > 0) + freq_index = static_cast(mean_diff / precision) + radius; + else + freq_index = static_cast(mean_diff / precision) - 1 + radius; + if (freq_index <= 0) { + freq_intervals[0]++; + } else if (freq_index >= static_cast(freq_intervals.size())) { + freq_intervals[freq_intervals.size() - 1]++; + } else { + freq_intervals[freq_index]++; + } + offset_count += sample_distance; + if (offset_count >= r3) { + n2_count++; + if (n2_count == r2) { + n1_count++; + n2_count = 1; + data_pos += r3; + } + offset_count_2 = (n1_count + n2_count) % sample_distance; + data_pos += (r3 + sample_distance - offset_count) + (sample_distance - offset_count_2); + offset_count = (sample_distance - offset_count_2); + if (offset_count == 0) offset_count++; + } else + data_pos += sample_distance; + sample_count++; + } + pred_freq = freq_count * 1.0 / sample_count; + mean_guess = mean_rough; + mean_freq = estimate_mean_freq_and_position(freq_intervals, precision, sample_count, mean_guess); + return estimate_quantization_intervals(intervals, sample_count); +} +} // namespace SZ3 + +#endif diff --git a/include/SZ3/utils/Sample.hpp b/include/SZ3/utils/Sample.hpp new file mode 100644 index 00000000..4072c969 --- /dev/null +++ b/include/SZ3/utils/Sample.hpp @@ -0,0 +1,294 @@ +#ifndef SZ3_SAMPLE_HPP +#define SZ3_SAMPLE_HPP + +#include +#include + +#include "SZ3/def.hpp" + +namespace SZ3 { +template +inline void profiling_block(T* data, std::vector& dims, std::vector>& starts, + size_t block_size, double abseb, size_t stride = 4) { + assert(dims.size() == N); + if (stride == 0) stride = block_size; + if constexpr (N == 4) { + size_t dimx = dims[0], dimy = dims[1], dimz = dims[2], dimw = dims[3], dimyzw = dimy * dimz * dimw, + dimzw = dimz * dimw; + if (dimx < block_size || dimy < block_size || dimz < block_size || dimw < block_size) { + return; + } + for (size_t i = 0; i < dimx - block_size; i += block_size) { + for (size_t j = 0; j < dimy - block_size; j += block_size) { + for (size_t k = 0; k < dimz - block_size; k += block_size) { + for (size_t l = 0; l < dimw - block_size; l += block_size) { + size_t start_idx = i * dimyzw + j * dimzw + k * dimw + l; + T min = data[start_idx]; + T max = data[start_idx]; + for (size_t ii = 0; ii <= block_size; ii += stride) { + for (size_t jj = 0; jj <= block_size; jj += stride) { + for (size_t kk = 0; kk <= block_size; kk += stride) { + for (size_t ll = 0; ll <= block_size; ll += stride) { + size_t cur_idx = start_idx + ii * dimyzw + jj * dimzw + kk * dimw + ll; + T cur_value = data[cur_idx]; + if (cur_value < min) + min = cur_value; + else if (cur_value > max) + max = cur_value; + } + } + } + } + if (max - min > abseb) { + size_t a[4] = {i, j, k, l}; + starts.push_back(std::vector(a, a + 4)); + } + } + } + } + } + } else if constexpr (N == 3) { + size_t dimx = dims[0], dimy = dims[1], dimz = dims[2], dimyz = dimy * dimz; + if (dimx < block_size || dimy < block_size || dimz < block_size) { + return; + } + for (size_t i = 0; i < dimx - block_size; i += block_size) { + for (size_t j = 0; j < dimy - block_size; j += block_size) { + for (size_t k = 0; k < dimz - block_size; k += block_size) { + size_t start_idx = i * dimyz + j * dimz + k; + T min = data[start_idx]; + T max = data[start_idx]; + for (size_t ii = 0; ii <= block_size; ii += stride) { + for (size_t jj = 0; jj <= block_size; jj += stride) { + for (size_t kk = 0; kk <= block_size; kk += stride) { + size_t cur_idx = start_idx + ii * dimyz + jj * dimz + kk; + T cur_value = data[cur_idx]; + if (cur_value < min) + min = cur_value; + else if (cur_value > max) + max = cur_value; + } + } + } + if (max - min > abseb) { + size_t a[3] = {i, j, k}; + starts.push_back(std::vector(a, a + 3)); + } + } + } + } + } else if constexpr (N == 2) { + size_t dimx = dims[0], dimy = dims[1]; + if (dimx < block_size || dimy < block_size) { + return; + } + for (size_t i = 0; i < dimx - block_size; i += block_size) { + for (size_t j = 0; j < dimy - block_size; j += block_size) { + size_t start_idx = i * dimy + j; + T min = data[start_idx]; + T max = data[start_idx]; + for (size_t ii = 0; ii <= block_size; ii += stride) { + for (size_t jj = 0; jj <= block_size; jj += stride) { + size_t cur_idx = start_idx + ii * dimy + jj; + T cur_value = data[cur_idx]; + if (cur_value < min) + min = cur_value; + else if (cur_value > max) + max = cur_value; + } + } + if (max - min > abseb) { + size_t a[2] = {i, j}; + starts.push_back(std::vector(a, a + 2)); + } + } + } + } else { + size_t dimx = dims[0]; + if (dimx < block_size) { + return; + } + for (size_t i = 0; i < dimx - block_size; i += block_size) { + size_t start_idx = i; + T min = data[start_idx]; + T max = data[start_idx]; + for (size_t ii = 0; ii <= block_size; ii += stride) { + size_t cur_idx = start_idx + ii; + T cur_value = data[cur_idx]; + if (cur_value < min) + min = cur_value; + else if (cur_value > max) + max = cur_value; + } + if (max - min > abseb) { + size_t a[1] = {i}; + starts.push_back(std::vector(a, a + 1)); + } + } + } +} + +template +inline void sample_blocks(T* data, std::vector& sampling_data, std::vector& dims, + std::vector& starts, size_t block_size) { + assert(dims.size() == N); + assert(starts.size() == N); + if constexpr (N == 4) { + if (dims[0] < block_size || dims[1] < block_size || dims[2] < block_size || dims[3] < block_size) { + return; + } + size_t sample_num = block_size * block_size * block_size * block_size; + sampling_data.resize(sample_num, 0); + size_t startx = starts[0], starty = starts[1], startz = starts[2], startw = starts[3], dimy = dims[1], + dimz = dims[2], dimw = dims[3]; + size_t cubic_block_size = block_size * block_size * block_size, square_block_size = block_size * block_size, + dimyzw = dimy * dimz * dimw, dimzw = dimz * dimw; + for (size_t i = 0; i < block_size; i++) { + for (size_t j = 0; j < block_size; j++) { + for (size_t k = 0; k < block_size; k++) { + for (size_t l = 0; l < block_size; l++) { + size_t sample_idx = i * cubic_block_size + j * square_block_size + k * block_size + l; + size_t idx = (i + startx) * dimyzw + (j + starty) * dimzw + (k + startz) * dimw + (l + startw); + sampling_data[sample_idx] = data[idx]; + } + } + } + } + } else if constexpr (N == 3) { + if (dims[0] < block_size || dims[1] < block_size || dims[2] < block_size) { + return; + } + size_t sample_num = block_size * block_size * block_size; + sampling_data.resize(sample_num, 0); + size_t startx = starts[0], starty = starts[1], startz = starts[2], dimy = dims[1], dimz = dims[2]; + size_t square_block_size = block_size * block_size, dimyz = dimy * dimz; + for (size_t i = 0; i < block_size; i++) { + for (size_t j = 0; j < block_size; j++) { + for (size_t k = 0; k < block_size; k++) { + size_t sample_idx = i * square_block_size + j * block_size + k; + size_t idx = (i + startx) * dimyz + (j + starty) * dimz + k + startz; + sampling_data[sample_idx] = data[idx]; + } + } + } + } else if constexpr (N == 2) { + if (dims[0] < block_size || dims[1] < block_size) { + return; + } + size_t sample_num = block_size * block_size; + sampling_data.resize(sample_num, 0); + size_t startx = starts[0], starty = starts[1], dimy = dims[1]; + for (size_t i = 0; i < block_size; i++) { + for (size_t j = 0; j < block_size; j++) { + size_t sample_idx = i * block_size + j; + size_t idx = (i + startx) * dimy + (j + starty); + sampling_data[sample_idx] = data[idx]; + } + } + } else if constexpr (N == 1) { + if (dims[0] < block_size) { + return; + } + size_t sample_num = block_size; + sampling_data.resize(sample_num, 0); + size_t startx = starts[0]; + for (size_t i = 0; i < block_size; i++) { + size_t sample_idx = i; + size_t idx = (i + startx); + sampling_data[sample_idx] = data[idx]; + } + } +} + +template +void sampleBlocks(T* data, std::vector& dims, size_t sampleBlockSize, + std::vector>& sampled_blocks, double sample_rate, int profiling, + std::vector>& starts) { + for (uint i = 0; i < N; i++) { + if (dims[i] < sampleBlockSize) { + return; + } + } + for (uint i = 0; i < sampled_blocks.size(); i++) { + std::vector().swap(sampled_blocks[i]); + } + std::vector>().swap(sampled_blocks); + for (uint i = 0; i < sampled_blocks.size(); i++) { + std::vector().swap(sampled_blocks[i]); + } + std::vector>().swap(sampled_blocks); + size_t totalblock_num = 1; + for (uint i = 0; i < N; i++) { + totalblock_num *= static_cast((dims[i] - 1) / sampleBlockSize); + } + size_t idx = 0; + if (profiling) { + size_t num_filtered_blocks = starts.size(); + size_t sample_stride = static_cast(num_filtered_blocks / (totalblock_num * sample_rate)); + if (sample_stride <= 0) sample_stride = 1; + for (size_t i = 0; i < num_filtered_blocks; i += sample_stride) { + std::vector s_block; + sample_blocks(data, s_block, dims, starts[i], sampleBlockSize + 1); + sampled_blocks.push_back(s_block); + } + } else { + size_t sample_stride = static_cast(1.0 / sample_rate); + if (sample_stride <= 0) sample_stride = 1; + if constexpr (N == 1) { + for (size_t x_start = 0; x_start < dims[0] - sampleBlockSize; x_start += sampleBlockSize) { + if (idx % sample_stride == 0) { + std::vector block_start{x_start}; + std::vector s_block; + sample_blocks(data, s_block, dims, block_start, sampleBlockSize + 1); + sampled_blocks.push_back(s_block); + } + idx += 1; + } + } else if constexpr (N == 2) { + for (size_t x_start = 0; x_start < dims[0] - sampleBlockSize; x_start += sampleBlockSize) { + for (size_t y_start = 0; y_start < dims[1] - sampleBlockSize; y_start += sampleBlockSize) { + if (idx % sample_stride == 0) { + std::vector block_start{x_start, y_start}; + std::vector s_block; + sample_blocks(data, s_block, dims, block_start, sampleBlockSize + 1); + sampled_blocks.push_back(s_block); + } + idx += 1; + } + } + } else if constexpr (N == 3) { + for (size_t x_start = 0; x_start < dims[0] - sampleBlockSize; x_start += sampleBlockSize) { + for (size_t y_start = 0; y_start < dims[1] - sampleBlockSize; y_start += sampleBlockSize) { + for (size_t z_start = 0; z_start < dims[2] - sampleBlockSize; z_start += sampleBlockSize) { + if (idx % sample_stride == 0) { + std::vector block_start{x_start, y_start, z_start}; + std::vector s_block; + sample_blocks(data, s_block, dims, block_start, sampleBlockSize + 1); + sampled_blocks.push_back(s_block); + } + idx += 1; + } + } + } + } else if constexpr (N == 4) { + for (size_t x_start = 0; x_start < dims[0] - sampleBlockSize; x_start += sampleBlockSize) { + for (size_t y_start = 0; y_start < dims[1] - sampleBlockSize; y_start += sampleBlockSize) { + for (size_t z_start = 0; z_start < dims[2] - sampleBlockSize; z_start += sampleBlockSize) { + for (size_t w_start = 0; w_start < dims[3] - sampleBlockSize; w_start += sampleBlockSize) { + if (idx % sample_stride == 0) { + std::vector block_start{x_start, y_start, z_start, w_start}; + std::vector s_block; + sample_blocks(data, s_block, dims, block_start, sampleBlockSize + 1); + sampled_blocks.push_back(s_block); + } + idx += 1; + } + } + } + } + } + } +} +} // namespace SZ3 + +#endif diff --git a/include/SZ3/utils/Statistic.hpp b/include/SZ3/utils/Statistic.hpp new file mode 100644 index 00000000..3de6bf40 --- /dev/null +++ b/include/SZ3/utils/Statistic.hpp @@ -0,0 +1,157 @@ +// +// Created by Kai Zhao on 4/20/20. +// + +#ifndef SZ3_STATISTIC_HPP +#define SZ3_STATISTIC_HPP + +#include +#include +#include + +#include "Config.hpp" + +namespace SZ3 { +template +T data_range(const T *data, size_t num) { + T max = data[0]; + T min = data[0]; + for (size_t i = 1; i < num; i++) { + if (max < data[i]) max = data[i]; + if (min > data[i]) min = data[i]; + } + return max - min; +} + +inline int factorial(int n) { return (n == 0) || (n == 1) ? 1 : n * factorial(n - 1); } + +inline double computeABSErrBoundFromPSNR(double psnr, double threshold, double value_range) { + double v1 = psnr + 10 * log10(1 - 2.0 / 3.0 * threshold); + double v2 = v1 / (-20); + double v3 = pow(10, v2); + return value_range * v3; +} + +template +void calAbsErrorBound(Config &conf, const T *data, T range = 0) { + if (conf.errorBoundMode != EB_ABS) { + if (conf.errorBoundMode == EB_REL) { + conf.errorBoundMode = EB_ABS; + conf.absErrorBound = conf.relErrorBound * ((range > 0) ? range : data_range(data, conf.num)); + } else if (conf.errorBoundMode == EB_PSNR) { + conf.errorBoundMode = EB_ABS; + conf.absErrorBound = computeABSErrBoundFromPSNR(conf.psnrErrorBound, 0.99, + ((range > 0) ? range : data_range(data, conf.num))); + } else if (conf.errorBoundMode == EB_L2NORM) { + conf.errorBoundMode = EB_ABS; + conf.absErrorBound = sqrt(3.0 / conf.num) * conf.l2normErrorBound; + } else if (conf.errorBoundMode == EB_ABS_AND_REL) { + conf.errorBoundMode = EB_ABS; + conf.absErrorBound = + std::min(conf.absErrorBound, conf.relErrorBound * ((range > 0) ? range : data_range(data, conf.num))); + } else if (conf.errorBoundMode == EB_ABS_OR_REL) { + conf.errorBoundMode = EB_ABS; + conf.absErrorBound = + std::max(conf.absErrorBound, conf.relErrorBound * ((range > 0) ? range : data_range(data, conf.num))); + } else { + throw std::invalid_argument("Error bound mode not supported"); + } + } +} + +template +double autocorrelation1DLag1(const Type *data, size_t numOfElem, Type avg) { + double cov = 0; + for (size_t i = 0; i < numOfElem; i++) { + cov += (data[i] - avg) * (data[i] - avg); + } + cov = cov / numOfElem; + + if (cov == 0) { + return 0; + } else { + int delta = 1; + double sum = 0; + + for (size_t i = 0; i < numOfElem - delta; i++) { + sum += (data[i] - avg) * (data[i + delta] - avg); + } + return sum / (numOfElem - delta) / cov; + } +} + +template +void verify(Type *ori_data, Type *data, size_t num_elements, double &psnr, double &nrmse, double &max_diff) { + size_t i = 0; + double Max = ori_data[0]; + double Min = ori_data[0]; + max_diff = fabs(data[0] - ori_data[0]); + double maxpw_relerr = 0; + double sum1 = 0, sum2 = 0, l2sum = 0; + for (i = 0; i < num_elements; i++) { + sum1 += ori_data[i]; + sum2 += data[i]; + l2sum += data[i] * data[i]; + } + double mean1 = sum1 / num_elements; + double mean2 = sum2 / num_elements; + + double sum3 = 0, sum4 = 0; + double sum = 0, prodSum = 0, relerr = 0; + + double *diff = static_cast(malloc(num_elements * sizeof(double))); + + for (i = 0; i < num_elements; i++) { + diff[i] = data[i] - ori_data[i]; + if (Max < ori_data[i]) Max = ori_data[i]; + if (Min > ori_data[i]) Min = ori_data[i]; + double err = fabs(data[i] - ori_data[i]); + if (ori_data[i] != 0) { + relerr = err / fabs(ori_data[i]); + if (maxpw_relerr < relerr) maxpw_relerr = relerr; + } + + if (max_diff < err) max_diff = err; + prodSum += (ori_data[i] - mean1) * (data[i] - mean2); + sum3 += (ori_data[i] - mean1) * (ori_data[i] - mean1); + sum4 += (data[i] - mean2) * (data[i] - mean2); + sum += err * err; + } + double std1 = sqrt(sum3 / num_elements); + double std2 = sqrt(sum4 / num_elements); + double ee = prodSum / num_elements; + double acEff = ee / std1 / std2; + + double mse = sum / num_elements; + double range = Max - Min; + psnr = 20 * log10(range) - 10 * log10(mse); + nrmse = sqrt(mse) / range; + + double normErr = sqrt(sum); + double normErr_norm = normErr / sqrt(l2sum); + + printf("Min=%.20G, Max=%.20G, range=%.20G\n", Min, Max, range); + printf("Max absolute error = %.2G\n", max_diff); + printf("Max relative error = %.2G\n", max_diff / (Max - Min)); + printf("Max pw relative error = %.2G\n", maxpw_relerr); + printf("PSNR = %f, NRMSE= %.10G\n", psnr, nrmse); + printf("normError = %f, normErr_norm = %f\n", normErr, normErr_norm); + printf("acEff=%f\n", acEff); + // printf("errAutoCorr=%.10f\n", autocorrelation1DLag1(diff, num_elements, diff_sum / num_elements)); + free(diff); +} + +template +void verify(Type *ori_data, Type *data, size_t num_elements) { + double psnr, nrmse, max_diff; + verify(ori_data, data, num_elements, psnr, nrmse, max_diff); +} + +template +void verify(Type *ori_data, Type *data, size_t num_elements, double &psnr, double &nrmse) { + double max_diff; + verify(ori_data, data, num_elements, psnr, nrmse, max_diff); +} +} // namespace SZ3 + +#endif diff --git a/include/SZ3/utils/Timer.hpp b/include/SZ3/utils/Timer.hpp new file mode 100644 index 00000000..5e278b30 --- /dev/null +++ b/include/SZ3/utils/Timer.hpp @@ -0,0 +1,43 @@ +// +// Created by Kai Zhao on 10/30/20. +// + +#ifndef SZ3_TIMER_HPP +#define SZ3_TIMER_HPP + +#include +#include +#include + +namespace SZ3 { +class Timer { + public: + Timer() = default; + + Timer(bool initstart) { + if (initstart) { + start(); + } + } + + void start() { begin = std::chrono::steady_clock::now(); } + + double stop() { + end = std::chrono::steady_clock::now(); + return std::chrono::duration(end - begin).count(); + } + + double stop([[maybe_unused]] const std::string &msg) { + double seconds = stop(); +#if defined(SZ3_DEBUG_TIMINGS) && SZ3_DEBUG_TIMINGS + std::cout << msg << " time = " << seconds << "s" << std::endl; +#endif + return seconds; + } + + private: + std::chrono::time_point begin, end; +}; +} // namespace SZ3 + +#endif // SZ_TIMER_HPP diff --git a/include/SZ3/version.hpp.in b/include/SZ3/version.hpp.in new file mode 100644 index 00000000..21e09d76 --- /dev/null +++ b/include/SZ3/version.hpp.in @@ -0,0 +1,40 @@ +// +// DO NOT MODIFY version.hpp, it is automatically generated by cmake +// Change the version numbers in CMakeLists.txt +// + +#ifndef SZ3_VERSION_HPP +#define SZ3_VERSION_HPP +#include +#include +#include + +#define SZ3_MAGIC_NUMBER 0xF342F310 + +#define SZ3_NAME "@PROJECT_NAME@" +#define SZ3_VER "@PROJECT_VERSION@" +#define SZ3_VER_MAJOR @PROJECT_VERSION_MAJOR@ +#define SZ3_VER_MINOR @PROJECT_VERSION_MINOR@ +#define SZ3_VER_PATCH @PROJECT_VERSION_PATCH@ +#define SZ3_VER_TWEAK @PROJECT_VERSION_TWEAK@ + +#define SZ3_DATA_VER "@SZ3_DATA_VERSION@" +// The oldest data version this build reads. +#define SZ3_DATA_VER_OLDEST "3.3.2" + +inline uint32_t versionInt(const std::string& version) { + uint32_t major = 0, minor = 0, patch = 0; + char dot; + std::stringstream ss(version); + ss >> major >> dot >> minor >> dot >> patch; + return (major << 24) | (minor << 16) | (patch<<8); +} + +inline std::string versionStr(uint32_t version) { + uint32_t major = (version >> 24) & 0xFF; + uint32_t minor = (version >> 16) & 0xFF; + uint32_t patch = (version >> 8) & 0xFF; + return std::to_string(major) + "." + std::to_string(minor) + "." + std::to_string(patch); +} + +#endif //SZ3_VERSION_HPP diff --git a/tools/H5Z-SZ3/CMakeLists.txt b/tools/H5Z-SZ3/CMakeLists.txt new file mode 100644 index 00000000..be17db06 --- /dev/null +++ b/tools/H5Z-SZ3/CMakeLists.txt @@ -0,0 +1,75 @@ +#Either use: +#find_package(HDF5 1.12.1 REQUIRED) +#or manually set the hdf5 paths: +#set(HDF5_LIBRARIES "/path/to/hdf5-1.12.1/hdf5/lib/libhdf5.so") +#set(HDF5_INCLUDE_DIRS "/path/to/hdf5-1.12.1/hdf5/include") + +find_package(HDF5 COMPONENTS C REQUIRED) + +#message("HDF5LIB PATH:" ${HDF5_LIBRARIES}) +#message("HDF5INCLUDE PATH:" ${HDF5_INCLUDE_DIRS}) + +add_library(hdf5sz3 + src/H5Z_SZ3.cpp +) +add_library(SZ3::hdf5sz3 ALIAS hdf5sz3) +# Export only what HDF5SZ3_EXPORT marks: an exported SZ3 internal can be taken over by another SZ3 +# copy loaded in the same process. +set_target_properties(hdf5sz3 PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON POSITION_INDEPENDENT_CODE ON) +if (NOT BUILD_SHARED_LIBS) + # Otherwise H5Z_SZ3.hpp marks the functions of a static hdf5sz3 dllimport on Windows. + target_compile_definitions(hdf5sz3 PUBLIC HDF5SZ3_STATIC) +endif () + +# SZ3core only: SZ3::SZ3 has OpenMP only where the consumer finds it, and the filter must link everywhere. +target_link_libraries(hdf5sz3 + PUBLIC SZ3core HDF5::HDF5 +) +# Do not add an MPI search: the filter calls no MPI function, and a parallel HDF5 brings its own. + +# Keep this: without a RUNPATH the installed library cannot find a conda, Spack or module HDF5. +set_target_properties(hdf5sz3 PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) +# SZ3 changes these functions' signatures and SZ3::Config's layout only in a new major.minor. +set_target_properties(hdf5sz3 PROPERTIES VERSION ${SZ3_VERSION} SOVERSION ${SZ3_VERSION_MAJOR}.${SZ3_VERSION_MINOR}) + +target_include_directories(hdf5sz3 + PUBLIC + # BUILD_INTERFACE only: exported, this machine's HDF5 headers would outrank a consumer's. + # Quoted, or CMake reads the leading $ as a relative path and prepends the source directory. + "$" + "$" + "$" +) + +if (BUILD_SZ3_BINARY) + add_subdirectory(tools) +endif () +# Only at top level, as SZ3's own tests: a parent's BUILD_TESTING is for its own tests. +if (BUILD_TESTING AND SZ3_IS_TOP_LEVEL) + add_subdirectory(test) +endif () + +if (SZ3_INSTALL) + install(TARGETS hdf5sz3 EXPORT HDF5SZ3 + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ) + + # A copy for HDF5 to find on its own. Its own directory, because HDF5 dlopens every lib*.so on + # HDF5_PLUGIN_PATH. + set(H5Z_SZ3_PLUGIN_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/plugin" CACHE STRING + "Directory to install the HDF5 filter plugin copy into; empty installs no copy") + if (BUILD_SHARED_LIBS AND H5Z_SZ3_PLUGIN_INSTALL_DIR) + install(TARGETS hdf5sz3 + LIBRARY DESTINATION ${H5Z_SZ3_PLUGIN_INSTALL_DIR} + RUNTIME DESTINATION ${H5Z_SZ3_PLUGIN_INSTALL_DIR} + # The Windows import library belongs beside the real one, not where HDF5 scans. + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + endif () + + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hdf5_sz3) + install(EXPORT HDF5SZ3 NAMESPACE SZ3:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3) +endif () +export(TARGETS hdf5sz3 FILE HDF5SZ3.cmake NAMESPACE SZ3::) diff --git a/tools/H5Z-SZ3/README.md b/tools/H5Z-SZ3/README.md new file mode 100644 index 00000000..6b702aee --- /dev/null +++ b/tools/H5Z-SZ3/README.md @@ -0,0 +1,135 @@ +# H5Z-SZ3 Filter Integration + +The H5Z-SZ3 filter integrates the SZ3 compression library with HDF5, providing an efficient way to compress and decompress data within HDF5 files. + +Use the filter from SZ3 3.4.0 or later. + +## Table of Contents +- [Installation](#installation) +- [H5Z-SZ3 cd_values](#h5z-sz3-cd_values) +- [Usage](#usage) + - [HDF5 Executables](#hdf5-executables) + - [Python (h5py)](#python-h5py) + - [C/C++](#cc) +- [Using the filter in an application](#using-the-filter-in-an-application) +- [Rules for the filter's code](#rules-for-the-filters-code) + +## Installation + +### Step 1: Build and Install SZ3 +Compile and install SZ3 with the H5Z-SZ3 filter enabled: +```bash +cmake -S . -B build -DBUILD_H5Z_FILTER=ON -DCMAKE_INSTALL_PREFIX= +cmake --build build -j && cmake --install build +``` +A filter built against HDF5 1.14.5 or newer needs HDF5 1.14.5 or newer at run time. + +### Step 2: Configure Environment +Installing puts a copy of the filter in `/lib/plugin` (`H5Z_SZ3_PLUGIN_INSTALL_DIR`; empty for no copy), a +directory that holds nothing else. Point `HDF5_PLUGIN_PATH` at that, not at `/lib` or +`/bin`: HDF5 dlopens every `lib*.so` in each directory on the path, and every `*.dll` on +Windows. The variable also replaces HDF5's own compiled-in plugin directory rather than adding to +it, so list that too if other filters are in use. +```bash +export HDF5_PLUGIN_PATH=/lib/plugin +``` +An application that ships the plugin with itself can call `H5PLprepend("")` at +startup instead and set nothing; one that links `SZ3::hdf5sz3` can call +`H5Zregister(H5PLget_plugin_info())` (before any `H5Zfilter_avail`, which may load another SZ3 plugin). +On Windows it still has to find the library: the install puts `hdf5sz3.dll` (`libhdf5sz3.dll` under +MinGW) in `/bin` and only the import library in `/lib`, and there is no RPATH to +record either, so `/bin` has to be on `PATH` before the application runs. + +## H5Z-SZ3 cd_values +* HDF5 restricts the parameters that can be passed to filters through an integers array called `cd_values`. +* H5Z-SZ3 uses `cd_values` to pass the desired compression settings (e.g., algorithm, error bounds) to the compression process. `cd_values[0]` is the SZ3 data version, `(major << 24) | (minor << 16) | (patch << 8)` (`0x03040000` for 3.4.0), followed by the `Config` object serialized with `save()`. +* `cd_values` are read only when compressing, including appends; decompression reads the configuration from the compressed data. + +## Usage + +### HDF5 Executables + +Note: if the HDF5 in your system contains both `h5repack-shared` and `h5repack`, you need to use `h5repack-shared` for filters, as outlined in [H5Z-ZFP Issue #137](https://github.com/LLNL/H5Z-ZFP/issues/137). + +#### Compression + +**Method 1: Compression with default settings (see SZ3/utils/Config.hpp for defaults)** +```bash +h5repack-shared -f UD=32024,0 data.h5 data.sz3.h5 +``` + +**Method 2: Compression with customized settings in configuration file** + +Step 1: Generate H5Z-SZ3 cd_values from SZ3 configuration file: +```bash +cdvalueHelper -c sz3.config +``` +Step 2: Compress by h5repack-shared with generated cd_values: +```bash +h5repack-shared -f UD=[put cd_values here] data.h5 data.sz3.h5 +``` + +#### Decompression +```bash +h5repack-shared -f NONE data.sz3.h5 data.sz3_decompressed.h5 +``` + +### Python (h5py) +To use H5Z-SZ3 with h5py in Python: + +1. Ensure h5py is linked against the same HDF5 library used to build the SZ3 filter (typically system HDF5). +2. Set `HDF5_PLUGIN_PATH` to the directory containing `libhdf5sz3.so` or `libhdf5sz3.dylib` **before importing h5py**. +3. Use h5py's compression interface with SZ3 filter ID 32024 and appropriate compression options. + +```python +import os +os.environ["HDF5_PLUGIN_PATH"] = "/path/to/prefix/lib/plugin" +import h5py +import numpy as np + +from cdvalueHelper import SZ3 +config = SZ3('ALGO_INTERP_LORENZO', absolute=1e-3) + +with h5py.File('data.h5', 'w') as f: + f.create_dataset('dataset', data=np.random.rand(100, 100), + compression=32024, compression_opts=tuple(config.cd_values)) +``` + +**Note on HDF5 Runtime Compatibility**: If h5py was installed with its bundled HDF5 (common on macOS), it may not load plugins built against system HDF5 due to library conflicts. To resolve: +- Rebuild h5py against system HDF5: `export HDF5_DIR=/path/to/system/hdf5 && pip install --force-reinstall --no-binary h5py h5py` +- Ensure HDF5 versions match between h5py and the plugin. + +### C/C++ +A C program sets the filter on a dataset creation property list with `H5Pset_sz3`, declared in `H5Z_SZ3.hpp`. A CMake project links `SZ3::hdf5sz3` and requires it with `find_package(SZ3 COMPONENTS hdf5sz3)`. See the examples `sz3ToHDF5.cpp` and `dsz3FromHDF5.cpp`. + +## Using the filter in an application + +### Setting and registering it +- Set the filter with `H5Pset_sz3(plist, cmprAlgo, errorBoundMode, abs, rel, psnr, l2norm)`, not by writing `Config::save()` into `cd_values` yourself. +- An application that links the filter registers it with `H5Zregister(H5PLget_plugin_info())`, before any `H5Zfilter_avail` (which may load another SZ3 plugin), and then needs no `HDF5_PLUGIN_PATH`. One that ships the plugin calls `H5PLprepend(dir)`. +- Register when the application opens any file, not only when it creates an SZ3 dataset, so that a program that only reads can read SZ3 data. A failed registration should not stop files without SZ3 data from opening. +- Registering replaces, for the whole process, an SZ3 filter loaded from `HDF5_PLUGIN_PATH`, also a newer one. State that choice where it is made. +- `h5repack` exits 0 and writes an uncompressed copy when it cannot load the filter. Check the output, for example with `h5dump -pH`, which shows the filter and its version. + +### Datatypes, chunks and other filters +- The dataset's type must be the host's float, double or 1, 2, 4 or 8-byte integer, in host byte order; other types are refused when the dataset is created. Chunks with more than 4 dimensions longer than 1 are refused too. +- For molecular-dynamics coordinates use `H5Z_SZ3_ALGO_BIOMD` (better with several frames per chunk). A `H5Z_SZ3_ALGO_BIOMD` chunk must have 3 as its last dimension, `(frames, atoms, 3)`; creating the dataset fails otherwise, and the error names the chunk shape. Set the chunk shape yourself (in h5py `chunks=(k, atoms, 3)`): h5py's automatic chunking splits the atoms and xyz. See [docs/molecular-dynamics.md](../../docs/molecular-dynamics.md). +- Do not put a filter that rearranges bytes, such as shuffle, before SZ3: SZ3 then compresses the rearranged bytes as values. The filter refuses a chunk whose size an earlier filter changed. +- SZ3 has no checksum; add `H5Pset_fletcher32` after `H5Pset_sz3` to detect damaged chunks. +- The filter keeps the flags it was set with. With `H5Z_FLAG_OPTIONAL` (h5py sets it), HDF5 stores a chunk raw when SZ3 fails on it. +- The filter does not use OpenMP. + +### Writing a chunk again and appending +- A chunk that is filtered and then written again is decompressed and compressed once more, and its error can then exceed the bound, except with `H5Z_SZ3_ALGO_NOPRED` with an absolute bound, and with `H5Z_SZ3_ALGO_BIOMD` with an absolute bound for float coordinates within B, 2^21 to 2^22 times the bound (2048 nm at 5e-4 nm, 256 nm at 1e-4 nm, 128 nm at 5e-5 nm, 32 nm at 1e-5 nm). A chunk with a coordinate past B, or with NaN or Inf, is stored losslessly, at a ratio of about 1.1 to 1.5 instead of 3 to 6; that also keeps a rewritten chunk within the bound. +- When appending frames, `H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS` alone does not stop recompression: once the dataset is extended past a chunk, HDF5 filters it, and the next write decompresses and recompresses it unless the chunk cache holds the whole chunk. Set `H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS` and give the dataset a chunk cache that holds a whole chunk (`H5Pset_chunk_cache`), also when the file is reopened. The last, partial chunk is then stored uncompressed at full chunk size. +- A test that writes and reads in the same open file reads HDF5's chunk cache and never decompresses. Close and reopen the file, and compare within the bound. To test that reading registers the filter, unregister it (`H5Zunregister`) first. + +## Rules for the filter's code +- `cd_values[0]` is `versionInt(SZ3_DATA_VER)`, followed by the `Config` bytes, little-endian. Tools that write `cd_values` (`cdvalueHelper.py`) follow the same layout. +- `H5Z_SZ3.hpp` is the only public header; everything outside its `__cplusplus` block stays plain C (CI compiles it as C99 with `-pedantic-errors -Werror`). The `H5Z_SZ3_*` constants are `static_assert`ed against the C++ enums. +- To read `cd_values` back, ask HDF5 for the count, allocate, then retrieve. +- Do not use `H5Zfilter_avail()` to decide anything about a property list: it answers for the library. Look the filter up on the list. +- `set_local` refuses `cd_values` it cannot read, rather than continuing with defaults. +- No exception may leave `H5Z_filter_sz3` or `set_local`: catch, push the message with `H5E_ERR_CLS` (`e.what()` through `"%s"`), and return 0 (filter) or a negative value (`set_local`). Print to stderr only when HDF5 has paused the error stack, never to stdout. +- The buffer returned to HDF5 comes from `malloc` and is freed if compression throws. +- The filter links `SZ3core` and does not use OpenMP. With `SZ3_DEBUG_TIMINGS` it still compiles every datatype `set_local` accepts. Test against HDF5 1.10, 1.14 and 2.x. diff --git a/tools/H5Z-SZ3/include/H5Z_SZ3.hpp b/tools/H5Z-SZ3/include/H5Z_SZ3.hpp new file mode 100644 index 00000000..e32fbffb --- /dev/null +++ b/tools/H5Z-SZ3/include/H5Z_SZ3.hpp @@ -0,0 +1,67 @@ +#ifndef SZ3_H5Z_SZ3_H +#define SZ3_H5Z_SZ3_H + +/* C programs include this header too: keep what is outside #ifdef __cplusplus plain C. */ + +#define H5Z_FILTER_SZ3 32024 + +#ifdef __cplusplus +#include "SZ3/api/sz.hpp" +#endif +#include "hdf5.h" +#include "H5PLextern.h" + +/* A static filter's functions stay hidden in the shared library it is linked into. */ +#if defined(HDF5SZ3_STATIC) + #define HDF5SZ3_EXPORT +#elif defined(_WIN32) + #if defined(hdf5sz3_EXPORTS) + #define HDF5SZ3_EXPORT __declspec(dllexport) + #else + #define HDF5SZ3_EXPORT __declspec(dllimport) + #endif +#else + #define HDF5SZ3_EXPORT __attribute__((visibility("default"))) +#endif + +/* SZ3::EB and SZ3::ALGO, repeated because C cannot see them. */ +#define H5Z_SZ3_EB_ABS 0 +#define H5Z_SZ3_EB_REL 1 +#define H5Z_SZ3_EB_PSNR 2 +#define H5Z_SZ3_EB_L2NORM 3 +#define H5Z_SZ3_EB_ABS_AND_REL 4 +#define H5Z_SZ3_EB_ABS_OR_REL 5 + +#define H5Z_SZ3_ALGO_LORENZO_REG 0 +#define H5Z_SZ3_ALGO_INTERP_LORENZO 1 +#define H5Z_SZ3_ALGO_INTERP 2 +#define H5Z_SZ3_ALGO_NOPRED 3 +#define H5Z_SZ3_ALGO_LOSSLESS 4 +#define H5Z_SZ3_ALGO_BIOMD 5 +/* Removed in 3.4.0; H5Pset_sz3 refuses it. Use H5Z_SZ3_ALGO_BIOMD. */ +#define H5Z_SZ3_ALGO_BIOMDXTC 6 + +#ifdef __cplusplus +extern "C" { +#endif + +/* Every bound must be finite and not negative; errorBoundMode decides which are used. Returns 1, or -1 with the reason on + * the HDF5 error stack. */ +HDF5SZ3_EXPORT herr_t H5Pset_sz3(hid_t propertyList, int cmprAlgo, int errorBoundMode, double absErrorBound, + double relErrorBound, double psnrErrorBound, double l2normErrorBound); + +#ifdef __cplusplus +/* Returns 1 on success, 0 on failure. */ +HDF5SZ3_EXPORT herr_t set_SZ3_conf_to_H5(hid_t propertyList, SZ3::Config &conf); + +/** + * @brief Load the SZ3 Config this property list carries. + * + * Returns 1 if the list carries the SZ3 filter, 0 if it does not, -1 if its cd_values cannot be read. + * conf changes only when 1 is returned and the filter has cd_values; without them it is left as it is. + */ +HDF5SZ3_EXPORT herr_t get_SZ3_conf_from_H5(hid_t propertyList, SZ3::Config &conf); +} +#endif + +#endif /* SZ3_H5Z_SZ3_H */ diff --git a/tools/H5Z-SZ3/src/H5Z_SZ3.cpp b/tools/H5Z-SZ3/src/H5Z_SZ3.cpp new file mode 100644 index 00000000..3278f6f0 --- /dev/null +++ b/tools/H5Z-SZ3/src/H5Z_SZ3.cpp @@ -0,0 +1,382 @@ +#include "H5Z_SZ3.hpp" + +#include +#include +#include +#include +#include +#include + +// The message is a printf format, so anything that is not a literal goes through "%s". +#define H5Z_SZ_PUSH_AND_GOTO(MAJ, MIN, RET, ...) \ + do { \ + H5Epush(H5E_DEFAULT, __FILE__, _funcname_, __LINE__, H5E_ERR_CLS, MAJ, MIN, __VA_ARGS__); \ + return RET; \ + } while (0) + +// The filter's own callbacks. HDF5 reaches them through H5Z_SZ3 below, so they stay out of the +// public header, where every consumer of the filter would see them declared and never defined. +extern "C" { +static herr_t H5Z_sz3_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id); + +static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, + size_t* buf_size, void** buf); +} + +const H5Z_class2_t H5Z_SZ3[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SZ3, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + // Written into every dataset that uses the filter, and quoted back by HDF5 when the filter + // is missing, so it has to say where to get it. + "H5Z-SZ3-" SZ3_VER " (data format " SZ3_DATA_VER "); see " + "https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3", + NULL, /* The "can apply" callback */ + H5Z_sz3_set_local, /* The "set local" callback */ + static_cast(H5Z_filter_sz3), /* The actual filter function */ +}}; + +HDF5SZ3_EXPORT H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_FILTER; } + +HDF5SZ3_EXPORT const void* H5PLget_plugin_info(void) { return H5Z_SZ3; } + +// set_SZ3_conf_to_H5() and get_sz3_conf_from_cdvalues() keep cd_values little-endian, even on big-endian hosts. +static void get_sz3_conf_from_cdvalues(const unsigned int* cd, size_t cd_nelmts, SZ3::Config& conf) { + std::vector buffer(cd_nelmts * sizeof(unsigned int)); + auto out = buffer.data(); + SZ3::write(cd, cd_nelmts, out); + const unsigned char* bytes = buffer.data(); + size_t len = buffer.size(); + { + // backward compatibility for v3.2.0 to v3.3.0; they store the magic number and the data version first. + if (cd_nelmts > 1 && cd[0] == SZ3_MAGIC_NUMBER) { + throw std::invalid_argument("SZ3 HDF5 filter: data is in SZ3 data format v" + versionStr(cd[1]) + + ", this build reads v" SZ3_DATA_VER); + } + } + { + // backward compatibility for v3.3.2; it stores no version, so the first byte is the Config's length, never 0. + if (bytes[0] != 0) { + conf.load(bytes, len); + return; + } + } + // Another data version may lay out the Config differently. + if (cd[0] < versionInt(SZ3_DATA_VER_OLDEST) || cd[0] > versionInt(SZ3_DATA_VER)) + throw std::invalid_argument("SZ3 HDF5 filter: data is in SZ3 data format v" + versionStr(cd[0]) + + ", this build reads v" SZ3_DATA_VER_OLDEST " to v" SZ3_DATA_VER); + bytes += sizeof(unsigned int); + len -= sizeof(unsigned int); + conf.load(bytes, len); +} + +// Do not use H5Zfilter_avail() here: it answers for the library, not for this property list. +static bool sz3_filter_on_plist(hid_t propertyList) { + const int nfilters = H5Pget_nfilters(propertyList); + for (int i = 0; i < nfilters; i++) { + unsigned int flags = 0; + size_t cd_nelmts = 0; + unsigned int cd_values[1] = {0}; + if (H5Z_FILTER_SZ3 == + H5Pget_filter2(propertyList, static_cast(i), &flags, &cd_nelmts, cd_values, 0, NULL, NULL)) { + return true; + } + } + return false; +} + +herr_t set_SZ3_conf_to_H5(hid_t propertyList, SZ3::Config& conf) { + static char const* _funcname_ = "set_SZ3_conf_to_H5"; + + std::vector cd_values(1 + conf.size_est(), 0); + cd_values[0] = versionInt(SZ3_DATA_VER); + auto pos = reinterpret_cast(cd_values.data() + 1); + cd_values.resize(1 + (conf.save(pos) + sizeof(unsigned int) - 1) / sizeof(unsigned int)); +#if SZ3_BIG_ENDIAN + for (size_t i = 1; i < cd_values.size(); i++) cd_values[i] = SZ3::byteswap(cd_values[i]); +#endif + size_t cd_nelmts = cd_values.size(); + + if (sz3_filter_on_plist(propertyList)) { + // Keep the flags the filter was set with: H5Z_FLAG_OPTIONAL lets HDF5 store a chunk raw when SZ3 fails. + unsigned int flags = H5Z_FLAG_MANDATORY; + size_t no_values = 0; + if (0 > H5Pget_filter_by_id(propertyList, H5Z_FILTER_SZ3, &flags, &no_values, NULL, 0, NULL, NULL) || + 0 > H5Pmodify_filter(propertyList, H5Z_FILTER_SZ3, flags, cd_nelmts, cd_values.data())) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "failed to modify cd_values"); + } + } else { + // filter not set, set filter. Notice that calling H5Pset_filter twice with the same filter id + // will cause unexpected errors for decompression + if (0 > H5Pset_filter(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values.data())) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "failed to modify cd_values"); + } + } + + return 1; +} + +herr_t H5Pset_sz3(hid_t propertyList, int cmprAlgo, int errorBoundMode, double absErrorBound, double relErrorBound, + double psnrErrorBound, double l2normErrorBound) { + static char const* _funcname_ = "H5Pset_sz3"; + static_assert(H5Z_SZ3_EB_ABS == SZ3::EB_ABS && H5Z_SZ3_EB_REL == SZ3::EB_REL && H5Z_SZ3_EB_PSNR == SZ3::EB_PSNR && + H5Z_SZ3_EB_L2NORM == SZ3::EB_L2NORM && H5Z_SZ3_EB_ABS_AND_REL == SZ3::EB_ABS_AND_REL && + H5Z_SZ3_EB_ABS_OR_REL == SZ3::EB_ABS_OR_REL, + "H5Z_SZ3.hpp error-bound modes must match SZ3::EB"); + static_assert(H5Z_SZ3_ALGO_LORENZO_REG == SZ3::ALGO_LORENZO_REG && + H5Z_SZ3_ALGO_INTERP_LORENZO == SZ3::ALGO_INTERP_LORENZO && + H5Z_SZ3_ALGO_INTERP == SZ3::ALGO_INTERP && H5Z_SZ3_ALGO_NOPRED == SZ3::ALGO_NOPRED && + H5Z_SZ3_ALGO_LOSSLESS == SZ3::ALGO_LOSSLESS && H5Z_SZ3_ALGO_BIOMD == SZ3::ALGO_BIOMD && + H5Z_SZ3_ALGO_BIOMDXTC == SZ3::ALGO_BIOMDXTC, + "H5Z_SZ3.hpp algorithms must match SZ3::ALGO"); + if (cmprAlgo < H5Z_SZ3_ALGO_LORENZO_REG || cmprAlgo > H5Z_SZ3_ALGO_BIOMDXTC) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "unknown SZ3 algorithm %d", cmprAlgo); + } + if (cmprAlgo == H5Z_SZ3_ALGO_BIOMDXTC) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, + "SZ3 ALGO_BIOMDXTC was removed in 3.4.0; use H5Z_SZ3_ALGO_BIOMD"); + } + if (errorBoundMode < H5Z_SZ3_EB_ABS || errorBoundMode > H5Z_SZ3_EB_ABS_OR_REL) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "unknown SZ3 error-bound mode %d", errorBoundMode); + } + for (double bound : {absErrorBound, relErrorBound, psnrErrorBound, l2normErrorBound}) { + if (!std::isfinite(bound) || bound < 0) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "SZ3 error bounds must be finite and not negative"); + } + } + SZ3::Config conf; + conf.cmprAlgo = static_cast(cmprAlgo); + conf.errorBoundMode = static_cast(errorBoundMode); + conf.absErrorBound = absErrorBound; + conf.relErrorBound = relErrorBound; + conf.psnrErrorBound = psnrErrorBound; + conf.l2normErrorBound = l2normErrorBound; + return set_SZ3_conf_to_H5(propertyList, conf) > 0 ? 1 : -1; +} + +herr_t get_SZ3_conf_from_H5(hid_t propertyList, SZ3::Config& conf) { + static char const* _funcname_ = "get_SZ3_conf_from_H5"; + + // H5Pget_filter_by_id fails when the list carries no SZ3 filter, which is not an error to report. + if (!sz3_filter_on_plist(propertyList)) { + return 0; + } + // Ask for the count before the values: a stored config carries the dataset's dimensions, so its + // length is not the caller's to guess, and HDF5 reports the real count even when it filled less. + size_t cd_nelmts = 0; + unsigned int probe[1] = {0}; + if (0 > H5Pget_filter_by_id(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, &cd_nelmts, probe, 0, NULL, NULL)) { + return -1; + } + if (cd_nelmts > 0) { + std::vector cd_values(cd_nelmts, 0); + if (0 > H5Pget_filter_by_id(propertyList, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, &cd_nelmts, cd_values.data(), 0, + NULL, NULL)) { + return -1; + } + try { + SZ3::Config loaded; + get_sz3_conf_from_cdvalues(cd_values.data(), cd_values.size(), loaded); + conf = loaded; + } catch (const std::exception& e) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "%s", e.what()); + } + } + return 1; +} + +static herr_t H5Z_sz3_set_local_impl(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id) { + // printf("start H5Z_sz3_set_local\n"); + + // printf("start in H5Z_sz3_set_local, dcpl_id = %d\n", dcpl_id); + static char const* _funcname_ = "H5Z_sz3_set_local"; + + // herr_t ret = H5Zregister(H5Z_SZ3); + + SZ3::Config conf; + if (get_SZ3_conf_from_H5(dcpl_id, conf) < 0) return -1; + + int ndims; + hsize_t dims_all[H5S_MAX_RANK]; + if (0 > (ndims = H5Sget_simple_extent_dims(chunk_space_id, dims_all, NULL))) + H5Z_SZ_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a data space"); + std::vector dims(dims_all, dims_all + ndims); + // The filter uses a chunk's bytes as host values, so the datatype has to be identical to a host + // type; H5Tequal also compares byte order, precision, offset and padding. + const std::pair host_types[] = {{H5T_NATIVE_FLOAT, SZ_FLOAT}, {H5T_NATIVE_DOUBLE, SZ_DOUBLE}, + {H5T_NATIVE_INT8, SZ_INT8}, {H5T_NATIVE_UINT8, SZ_UINT8}, + {H5T_NATIVE_INT16, SZ_INT16}, {H5T_NATIVE_UINT16, SZ_UINT16}, + {H5T_NATIVE_INT32, SZ_INT32}, {H5T_NATIVE_UINT32, SZ_UINT32}, + {H5T_NATIVE_INT64, SZ_INT64}, {H5T_NATIVE_UINT64, SZ_UINT64}}; + const auto host_type = std::find_if(std::begin(host_types), std::end(host_types), + [&](const auto& t) { return H5Tequal(type_id, t.first) > 0; }); + if (host_type == std::end(host_types)) + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, -1, + "datatype must be the host's float, double, or 1, 2, 4 or 8-byte integer"); + conf.dataType = host_type->second; + // update conf with dims + conf.setDims(std::begin(dims), std::end(dims)); + if (conf.N > 4) + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, "SZ3 compresses at most 4 dimensions longer than 1, not %d", + static_cast(conf.N)); + if (conf.cmprAlgo == SZ3::ALGO_BIOMD && (conf.N > 3 || conf.dims[conf.N - 1] != 3)) { + std::string chunk; + for (size_t d : dims) chunk += (chunk.empty() ? "" : ", ") + std::to_string(d); + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, -1, + "SZ3 ALGO_BIOMD needs chunks whose last dimension is 3, (frames, atoms, 3); these are (%s)", + chunk.c_str()); + } + // need to update magic number and data version, + // as the config may be from cd_values passed by users + conf.sz3MagicNumber = SZ3_MAGIC_NUMBER; + conf.sz3DataVer = versionInt(SZ3_DATA_VER); + + if (set_SZ3_conf_to_H5(dcpl_id, conf) <= 0) return -1; + return 1; +} + +template +void process_data(SZ3::Config& conf, void** buf, size_t* buf_size, size_t nbytes, bool is_decompress) { + if (is_decompress) { + // HDF5 frees what this returns, so it has to come from malloc. On null SZ_decompress would + // allocate with new[] instead, and that pairing is undefined. + std::unique_ptr processedData(static_cast(malloc(conf.num * sizeof(T))), &free); + if (!processedData) throw std::bad_alloc(); + T* decData = processedData.get(); + SZ_decompress(conf, static_cast(*buf), nbytes, decData); + free(*buf); + *buf = processedData.release(); + *buf_size = conf.num * sizeof(T); + } else { + if (nbytes != conf.num * sizeof(T)) + throw std::invalid_argument("SZ3 HDF5 filter: a filter before SZ3 changed the chunk's size"); + // The bound assumes the payload fits in the raw size, so leave headroom on top of it for + // algorithms whose output can reach or exceed that. + size_t cmpCap = std::max(SZ3::SZ_compress_size_bound(conf), sizeof(T) * conf.num * 2); + std::unique_ptr cmpData(static_cast(malloc(cmpCap)), &free); + if (!cmpData) throw std::bad_alloc(); + *buf_size = SZ_compress(conf, static_cast(*buf), cmpData.get(), cmpCap); + free(*buf); + *buf = cmpData.release(); + } +} + +/** + * https://docs.hdfgroup.org/hdf5/v1_14/_f_i_l_t_e_r.html + * The flags, cd_nelmts, and cd_values are the same as for the H5Pset_filter() function with the additional flag + * H5Z_FLAG_REVERSE which is set when the filter is called as part of the input pipeline. The input buffer is pointed to + * by *buf and has a total size of *buf_size bytes but only nbytes are valid data. The filter should perform the + * transformation in place if possible and return the number of valid bytes or zero for failure. If the transformation + * cannot be done in place then the filter should allocate a new buffer with malloc() and assign it to *buf, assigning + * the allocated size of that buffer to *buf_size. The old buffer should be freed by calling free(). + */ +static size_t H5Z_filter_sz3_impl(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], + size_t nbytes, size_t* buf_size, void** buf) { + // printf("start H5Z_filter_sz3\n"); + + if (cd_nelmts == 0) // this is special data such as string, which should not be treated as values. + return nbytes; + + bool is_decompress = flags & H5Z_FLAG_REVERSE; + SZ3::Config conf; + + if (is_decompress) { + // The chunk carries its own Config, which gives the element count and type to decompress. + uint32_t magic = 0, dataVer = 0; + uint64_t cmpDataSize = 0; + auto pos = static_cast(*buf); + if (nbytes >= 16) { + SZ3::read(magic, pos); + SZ3::read(dataVer, pos); + SZ3::read(cmpDataSize, pos); + } + if (magic != SZ3_MAGIC_NUMBER) { + { + // backward compatibility for v3.3.2; it stores chunks of fewer than 20 elements raw, with no header, + // so in fewer than 20 * 8 bytes. + if (nbytes < 20 * sizeof(double)) return nbytes; + } + throw std::invalid_argument("SZ3 HDF5 filter: chunk is not in a data format this SZ3 reads, or is damaged"); + } + if (dataVer < versionInt(SZ3_DATA_VER_OLDEST) || dataVer > versionInt(SZ3_DATA_VER)) + throw std::invalid_argument("SZ3 HDF5 filter: data is in SZ3 data format v" + versionStr(dataVer) + + ", this build reads v" SZ3_DATA_VER_OLDEST " to v" SZ3_DATA_VER + + (dataVer > versionInt(SZ3_DATA_VER) + ? "; upgrade SZ3 to read it" + : "; use SZ3 v" + versionStr(dataVer) + " to read it")); + if (cmpDataSize > nbytes - 16) throw std::invalid_argument("SZ3 HDF5 filter: chunk is truncated"); + size_t remaining = nbytes - 16 - cmpDataSize; + pos += cmpDataSize; + conf.load(pos, remaining); + } else { + get_sz3_conf_from_cdvalues(cd_values, cd_nelmts, conf); + } + + switch (conf.dataType) { + case SZ_FLOAT: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_DOUBLE: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_INT8: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_UINT8: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_INT16: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_UINT16: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_INT32: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_UINT32: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_INT64: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + case SZ_UINT64: + process_data(conf, buf, buf_size, nbytes, is_decompress); + break; + default: + throw std::invalid_argument("SZ3 HDF5 filter: unknown datatype in cd_values"); + } + return *buf_size; +} + +// HDF5 calls these from C, where an exception that escapes ends the application. Zero is how a +// filter reports failure; set_local uses a negative return. +static size_t H5Z_filter_sz3(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, + size_t* buf_size, void** buf) { + static char const* _funcname_ = "H5Z_filter_sz3"; + try { + return H5Z_filter_sz3_impl(flags, cd_nelmts, cd_values, nbytes, buf_size, buf); + } catch (const std::exception& e) { +#if H5_VERSION_GE(1, 14, 5) + // 1.14.5 and 1.14.6 pause the error stack around filters, which drops what is pushed below. Print only + // then: stdout would land in h5dump's output. + bool paused = false; + if (H5Eis_paused(H5E_DEFAULT, &paused) >= 0 && paused) fprintf(stderr, "%s\n", e.what()); +#endif + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_CALLBACK, 0, "%s", e.what()); + } catch (...) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_CALLBACK, 0, "unknown error"); + } +} + +static herr_t H5Z_sz3_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id) { + static char const* _funcname_ = "H5Z_sz3_set_local"; + try { + return H5Z_sz3_set_local_impl(dcpl_id, type_id, chunk_space_id); + } catch (const std::exception& e) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_CALLBACK, -1, "%s", e.what()); + } catch (...) { + H5Z_SZ_PUSH_AND_GOTO(H5E_PLINE, H5E_CALLBACK, -1, "unknown error"); + } +} diff --git a/tools/H5Z-SZ3/test/CMakeLists.txt b/tools/H5Z-SZ3/test/CMakeLists.txt new file mode 100644 index 00000000..a161bff6 --- /dev/null +++ b/tools/H5Z-SZ3/test/CMakeLists.txt @@ -0,0 +1,17 @@ +# CI installs the HDF5 tools and runs filterSuite.sh directly, so coverage does not depend +# on this registration. +find_program(H5REPACK_EXECUTABLE h5repack) +find_program(H5DUMP_EXECUTABLE h5dump) +find_program(H5LS_EXECUTABLE h5ls) +if (SZ3_INSTALL AND H5REPACK_EXECUTABLE AND H5DUMP_EXECUTABLE AND H5LS_EXECUTABLE) + get_filename_component(H5Z_SZ3_TOOL_DIR "${H5DUMP_EXECUTABLE}" DIRECTORY) + add_test(NAME SZ3_FilterSuite + COMMAND ${CMAKE_COMMAND} + -D SZ3_BUILD_DIR=${CMAKE_BINARY_DIR} + -D SZ3_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/_filtersuite + -D SZ3_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/filterSuite.sh + -D SZ3_H5BIN=${H5Z_SZ3_TOOL_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/runFilterSuite.cmake) +else () + message(STATUS "SZ3_FilterSuite not registered: h5repack, h5dump and h5ls are not all on PATH") +endif () diff --git a/tools/H5Z-SZ3/test/filterSuite.sh b/tools/H5Z-SZ3/test/filterSuite.sh new file mode 100755 index 00000000..29ab64b9 --- /dev/null +++ b/tools/H5Z-SZ3/test/filterSuite.sh @@ -0,0 +1,435 @@ +#!/bin/bash +# Every way an application or a user reaches the SZ3 HDF5 filter, asserted against an install tree. +# +# tools/H5Z-SZ3/test/filterSuite.sh [] +# +# CMAKE_PREFIX_PATH is passed through. Assert which path was taken, never just that the exit was 0. +set -u +PREFIX=$(cd "$1" && pwd) +HERE=$(cd "$(dirname "$0")" && pwd) +H5BIN=${2:-} +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +cd "$WORK" || exit 1 + +h5() { if [ -n "$H5BIN" ]; then echo "$H5BIN/$1"; else echo "$1"; fi; } +# Under MSYS2 and Cygwin, cmake and the HDF5 tools are native Windows programs while this shell +# deals in /d/... paths they cannot open. A prefix handed over unconverted is simply not searched, +# and cmake then finds SZ3 wherever else it can -- in CI, the build tree on PATH. +native() { if command -v cygpath > /dev/null 2>&1; then cygpath -m "$1"; else echo "$1"; fi; } +LIBDIR=$PREFIX/lib +[ -d "$LIBDIR" ] || LIBDIR=$PREFIX/lib64 +PLUGIN_DIR=$LIBDIR/plugin +NOPLUGIN=$WORK/no-such-plugin-dir +# HDF5 reads HDF5_PLUGIN_PATH itself, so it needs the same native form cmake does. +PLUGIN_PATH=$(native "$PLUGIN_DIR") +NOPLUGIN_PATH=$(native "$NOPLUGIN") +# Windows resolves a DLL through PATH and has no RPATH for an install tree to be recorded in, so +# /bin -- where the install rule puts the runtime artifact -- is the only way an application +# that links the shared filter reaches libhdf5sz3.dll. POSIX form: bash splits PATH on ':', and the +# MSYS2 runtime converts the whole variable when it spawns a native program. +export PATH="$PREFIX/bin:$PATH" + +# Every spelling the filter takes when it is shared: .so on Linux, .dylib on macOS, hdf5sz3.dll +# under MSVC and libhdf5sz3.dll under MinGW. An archive is deliberately not among them. +shared_filter_in() { + for f in "$1"/libhdf5sz3.so* "$1"/libhdf5sz3*.dylib "$1"/hdf5sz3.dll "$1"/libhdf5sz3.dll; do + [ -e "$f" ] && return 0 + done + return 1 +} +# The plugin is a shared object HDF5 dlopens. BUILD_SHARED_LIBS=OFF installs an archive instead, so +# every mode that goes through HDF5_PLUGIN_PATH, and the DT_NEEDED entry the loader would record, +# describe something this install tree does not contain. Reported as failures they say the filter is +# broken; they are skips, named and counted, and the total at the bottom is asserted. +HAVE_PLUGIN=0 +shared_filter_in "$PLUGIN_DIR" && HAVE_PLUGIN=1 +HAVE_SHARED=0 +shared_filter_in "$LIBDIR" && HAVE_SHARED=1 +shared_filter_in "$PREFIX/bin" && HAVE_SHARED=1 +# What decides the skip is the shape of the library, not the absence of the plugin file -- otherwise +# a plugin that failed to install would excuse itself. A shared filter with no plugin beside it is +# the install rule being wrong, and has to be read that way. +if [ "$HAVE_SHARED" = 1 ] && [ "$HAVE_PLUGIN" = 0 ]; then + echo "FATAL: $LIBDIR holds a shared hdf5sz3 but $PLUGIN_DIR holds nothing for HDF5 to load" + exit 1 +fi + +pass=0; fail=0; skip=0 +ok() { echo "PASS $1"; pass=$((pass+1)); } +bad() { echo "FAIL $1"; shift; for l in "$@"; do echo " $l"; done; fail=$((fail+1)); } +skipped() { echo "SKIP $1"; skip=$((skip+1)); } +# reason , then the names it covers +skip_all() { reason=$1; shift; for n in "$@"; do skipped "$n ($reason)"; done; } +# want +want() { if grep -qF "$2" "$3"; then ok "$1"; else bad "$1" "expected to find: $2" "got:" "$(head -5 "$3")"; fi; } +notwant() { if grep -qF "$2" "$3"; then bad "$1" "did not expect: $2"; else ok "$1"; fi; } + +# Required, not skipped: a missing tool must never look like a passing check. +missing= +for tool in h5repack h5dump h5ls; do + command -v "$(h5 $tool)" > /dev/null 2>&1 || missing="$missing $tool" +done +if [ -n "$missing" ]; then + echo "HDF5 command-line tools not found:$missing" + echo "Install them (Debian/Ubuntu: hdf5-tools) or pass their directory as the second argument." + exit 1 +fi + +echo "=== prefix $PREFIX ===" +"$(h5 h5dump)" --version 2>&1 | head -1 | sed 's/^/ /' + +# ---------------------------------------------------------------- fixtures +cat > gen.c <<'EOF' +/* The input fixture. No SZ3 anywhere in it. */ +#include +#include +#include +int main(int argc, char **argv) { + hsize_t dims[2] = {64, 64}, ch[2] = {16, 64}; + float d[64 * 64]; + for (int i = 0; i < 64 * 64; i++) d[i] = (float)sin(0.01 * i) * 100.0f; + hid_t f = H5Fcreate(argv[1], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + hid_t s = H5Screate_simple(2, dims, NULL); + hid_t p = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_chunk(p, 2, ch); + hid_t d2 = H5Dcreate2(f, "ds", H5T_NATIVE_FLOAT, s, H5P_DEFAULT, p, H5P_DEFAULT); + herr_t r = H5Dwrite(d2, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, d); + H5Dclose(d2); H5Pclose(p); H5Sclose(s); + + /* A side past 4096 makes the stored Config longer than a default-constructed one. */ + hsize_t wide[3] = {8192, 4, 4}; + float *w = malloc(8192 * 4 * 4 * sizeof(float)); + for (int i = 0; i < 8192 * 4 * 4; i++) w[i] = (float)sin(0.001 * i) * 100.0f; + hid_t s3 = H5Screate_simple(3, wide, NULL); + hid_t p3 = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_chunk(p3, 3, wide); + hid_t d3 = H5Dcreate2(f, "wide", H5T_NATIVE_FLOAT, s3, H5P_DEFAULT, p3, H5P_DEFAULT); + if (H5Dwrite(d3, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, w) < 0) r = -1; + H5Dclose(d3); H5Pclose(p3); H5Sclose(s3); free(w); H5Fclose(f); + return r < 0; +} +EOF +# A reader that links no SZ3 at all: the plugin is its only way in. +cat > read.c <<'EOF' +#include +#include +#include +int main(int argc, char **argv) { + float d[64 * 64]; + hid_t f = H5Fopen(argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); + if (f < 0) { printf("OPEN FAILED\n"); return 2; } + hid_t ds = H5Dopen2(f, "ds", H5P_DEFAULT); + if (H5Dread(ds, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, d) < 0) { + printf("READ FAILED\n"); return 1; + } + double m = 0; + for (int i = 0; i < 64 * 64; i++) { + double e = fabs((double)d[i] - (double)((float)sin(0.01 * i) * 100.0f)); + if (e > m) m = e; + } + if (m > 1e-3) { + printf("READ OUT OF BOUND maxerr %.6f\n", m); + return 1; + } + printf("READ OK maxerr %.6f\n", m); + return 0; +} +EOF +# The ways an application can reach the filter. argv[1] picks one; prepend takes a directory. +cat > app.c <<'EOF' +#include +#include +#include +int main(int argc, char **argv) { + if (argv[1][0] == 'r') printf("%s\n", H5Zregister(H5PLget_plugin_info()) < 0 ? "REGISTER FAILED" : "REGISTER OK"); + if (argv[1][0] == 'p' && argv[1][1] == 'r') + printf("%s\n", argc > 3 && H5PLprepend(argv[3]) >= 0 ? "PREPEND OK" : "PREPEND FAILED"); + printf("AVAIL %d\n", (int)H5Zfilter_avail(H5Z_FILTER_SZ3)); + float d[64 * 64]; + for (int i = 0; i < 64 * 64; i++) d[i] = (float)sin(0.01 * i) * 100.0f; + hsize_t dims[2] = {64, 64}, ch[2] = {16, 64}; + hid_t f = H5Fcreate(argv[2], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + hid_t s = H5Screate_simple(2, dims, NULL); + hid_t p = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_chunk(p, 2, ch); + SZ3::Config conf(16, 64); + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 1e-3; + set_SZ3_conf_to_H5(p, conf); + hid_t d2 = H5Dcreate2(f, "ds", H5T_NATIVE_FLOAT, s, H5P_DEFAULT, p, H5P_DEFAULT); + if (d2 < 0 || H5Dwrite(d2, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, d) < 0) { + printf("WRITE FAILED\n"); return 1; + } + H5Dclose(d2); H5Pclose(p); H5Sclose(s); H5Fclose(f); + printf("WRITE OK\n"); + return 0; +} +EOF +# The C interface. Keep it C: nothing else compiles the header as C. +cat > capi.c <<'EOF' +#include +#include +#include +int main(int argc, char **argv) { + float d[64 * 64], r[64 * 64]; + for (int i = 0; i < 64 * 64; i++) d[i] = (float)sin(0.01 * i) * 100.0f; + hsize_t dims[2] = {64, 64}, ch[2] = {16, 64}; + H5Zregister(H5PLget_plugin_info()); + hid_t p = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_chunk(p, 2, ch); + if (H5Pset_sz3(p, H5Z_SZ3_ALGO_INTERP_LORENZO, 99, 0.5, 0, 0, 0) < 0) printf("UNKNOWN MODE REFUSED\n"); + if (H5Pset_sz3(p, H5Z_SZ3_ALGO_BIOMDXTC, H5Z_SZ3_EB_ABS, 0.5, 0, 0, 0) < 0) printf("BIOMDXTC REFUSED\n"); + H5Pset_sz3(p, H5Z_SZ3_ALGO_INTERP_LORENZO, H5Z_SZ3_EB_ABS, 0.5, 0, 0, 0); + hid_t f = H5Fcreate(argv[1], H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + hid_t s = H5Screate_simple(2, dims, NULL); + hid_t ds = H5Dcreate2(f, "ds", H5T_NATIVE_FLOAT, s, H5P_DEFAULT, p, H5P_DEFAULT); + herr_t w = H5Dwrite(ds, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, d); + /* Types the filter cannot compress must fail at H5Dcreate, not come back as wrong data. */ + hsize_t three = 3; + hid_t cmp = H5Tcreate(H5T_COMPOUND, 8), arr = H5Tarray_create2(H5T_NATIVE_FLOAT, 1, &three); + hid_t i24 = H5Tcopy(H5T_NATIVE_INT), f128 = H5Tcopy(H5T_NATIVE_DOUBLE), swapped = H5Tcopy(H5T_NATIVE_FLOAT); + hid_t i24in32 = H5Tcopy(H5T_NATIVE_INT), fshort = H5Tcopy(H5T_NATIVE_FLOAT); + H5Tinsert(cmp, "x", 0, H5T_NATIVE_FLOAT); + H5Tinsert(cmp, "id", 4, H5T_NATIVE_INT); + H5Tset_precision(i24, 24); + H5Tset_size(i24, 3); + H5Tset_size(f128, 16); + H5Tset_order(swapped, H5Tget_order(H5T_NATIVE_FLOAT) == H5T_ORDER_LE ? H5T_ORDER_BE : H5T_ORDER_LE); + H5Tset_precision(i24in32, 24); + H5Tset_fields(fshort, 31, 23, 8, 1, 22); + const char *names[7] = {"compound", "array", "int24", "float128", "byteswapped", "int24in32", "floatshort"}; + hid_t types[7] = {cmp, arr, i24, f128, swapped, i24in32, fshort}; + H5Eset_auto2(H5E_DEFAULT, NULL, NULL); + for (int k = 0; k < 7; k++) { + hid_t bad = H5Dcreate2(f, names[k], types[k], s, H5P_DEFAULT, p, H5P_DEFAULT); + printf("%s %s\n", names[k], bad < 0 ? "REFUSED" : "ACCEPTED"); + if (bad >= 0) H5Dclose(bad); + H5Tclose(types[k]); + } + H5Dclose(ds); H5Pclose(p); H5Sclose(s); H5Fclose(f); + f = H5Fopen(argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); + ds = H5Dopen2(f, "ds", H5P_DEFAULT); + if (w < 0 || H5Dread(ds, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, r) < 0) { + printf("ROUND TRIP FAILED\n"); return 1; + } + double m = 0; + for (int i = 0; i < 64 * 64; i++) if (fabs((double)r[i] - d[i]) > m) m = fabs((double)r[i] - d[i]); + /* Under a tenth of the bound means the filter ran with some other bound, or did not run. */ + printf("maxerr %.6f\n%s\n", m, m > 0.05 && m <= 0.5 ? "BOUND OK" : "BOUND WRONG"); + return 0; +} +EOF + +printf '#include \nint main(void){return H5open() < 0;}\n' > noref.c +cat > CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(modes C CXX) +find_package(SZ3 REQUIRED) +find_package(HDF5 COMPONENTS C REQUIRED) +find_library(MATH_LIB m) +if (NOT MATH_LIB) + set(MATH_LIB "") +endif () +add_executable(gen gen.c) +add_executable(read read.c) +target_link_libraries(gen PRIVATE HDF5::HDF5 ${MATH_LIB}) +target_link_libraries(read PRIVATE HDF5::HDF5 ${MATH_LIB}) +add_executable(app app.c) +set_source_files_properties(app.c PROPERTIES LANGUAGE CXX) +target_link_libraries(app PRIVATE SZ3::SZ3 SZ3::hdf5sz3) +add_executable(capi capi.c) +target_link_libraries(capi PRIVATE SZ3::hdf5sz3 ${MATH_LIB}) +# The control for the DT_NEEDED check below. Without it that check passes vacuously. +add_executable(noref noref.c) +set_source_files_properties(noref.c PROPERTIES LANGUAGE CXX) +target_link_libraries(noref PRIVATE SZ3::SZ3 SZ3::hdf5sz3) +EOF +# When the HDF5 tools were given explicitly, build against that same HDF5. +CMPFX=$(native "$PREFIX") +[ -n "$H5BIN" ] && CMPFX="$CMPFX;$(native "$(cd "$H5BIN/.." && pwd)")" +[ -n "${CMAKE_PREFIX_PATH:-}" ] && CMPFX="$CMPFX;$CMAKE_PREFIX_PATH" +# A multi-config generator ignores CMAKE_BUILD_TYPE and writes the probes to b/, where the +# ./b/... below cannot find them. Naming the config and its output directory puts them at b/ on +# either generator kind. +cmake -S . -B b -DCMAKE_PREFIX_PATH="$CMPFX" -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE="$(native "$WORK")/b" > cmake.log 2>&1 \ + && cmake --build b --config Release -j 4 >> cmake.log 2>&1 \ + || { echo "FATAL: could not build the probes"; tail -20 cmake.log; exit 1; } +./b/gen plain.h5 || { echo "FATAL: could not write the fixture"; exit 1; } + +# The cd_values a user has to type. Mirrors SZ3::Config::save(); see cdvalueHelper.py. +CD_ABS="UD=32024,0,8,32,0,16777216,4054449152,1348619730,41023,256,0" +# What the filter stores in front of the Config: versionInt(SZ3_DATA_VER). +IFS=. read -r v1 v2 v3 <<< "$(sed -n 's/^#define SZ3_DATA_VER "\(.*\)"/\1/p' "$PREFIX/include/SZ3/version.hpp")" +CD_VER=$(( (v1 << 24) | (v2 << 16) | (v3 << 8) )) + +# ---------------------------------------------------------------- 1. h5repack, 2. h5dump / h5ls +# Both sections reach the filter only through HDF5_PLUGIN_PATH, and the h5dump section reads the +# file h5repack wrote, so neither says anything without a plugin to dlopen. +if [ "$HAVE_PLUGIN" = 1 ]; then +export HDF5_PLUGIN_PATH=$PLUGIN_PATH +"$(h5 h5repack)" -f "$CD_ABS" plain.h5 rp.h5 > rp.log 2>&1 +"$(h5 h5dump)" -pH rp.h5 > rp.head 2>&1 +want "h5repack-applies-sz3" "FILTER_ID 32024" rp.head +want "h5repack-records-version" "H5Z-SZ3-" rp.head +# CD_ABS has no version in front, as 3.3.2 wrote it; what gets stored starts with the data version +want "stored-cdvalues-carry-data-version" "PARAMS { $CD_VER " rp.head +./b/read rp.h5 > rp.read 2>&1 +want "h5repack-output-reads-back" "READ OK" rp.read +# The fixture's ds as the 3.3.2 filter stored it with CD_ABS. +./b/read "$(native "$HERE/sz3_data_3.3.2.h5")" > prev.read 2>&1 +want "previous-data-version-reads-back" "READ OK" prev.read + +# A chunk with a side past 4096 stores a Config longer than a default-constructed one measures. Read +# that back into a buffer sized from the default and the tail comes out as zeros -- quantbinCnt among +# them, which turns quantisation off and leaves the chunk near its raw size with nothing reported. +"$(h5 h5ls)" -v rp.h5/wide > wide.ls 2>&1 +wide_bytes=$(sed -n 's/.*Storage: *\([0-9]*\) logical bytes, *\([0-9]*\) allocated.*/\2/p' wide.ls | head -1) +if [ -n "$wide_bytes" ] && [ "$wide_bytes" -lt 131072 ]; then + ok "wide-chunk-still-compresses" +else + bad "wide-chunk-still-compresses" "524288 raw bytes stored as ${wide_bytes:-?}" "$(head -20 wide.ls)" +fi +"$(h5 h5repack)" -f NONE rp.h5 rp_none.h5 > strip.log 2>&1 +"$(h5 h5dump)" -pH rp_none.h5 > strip.head 2>&1 +notwant "h5repack-none-strips-sz3" "32024" strip.head +"$(h5 h5ls)" rp_none.h5 > strip.ls 2>&1 +want "h5repack-none-keeps-dataset" "ds" strip.ls + +# With the filter unreachable h5repack exits 0 and quietly writes an UNFILTERED copy. +export HDF5_PLUGIN_PATH=$NOPLUGIN_PATH +"$(h5 h5repack)" -f "$CD_ABS" plain.h5 rp_noplug.h5 > rpn.log 2>&1 +"$(h5 h5dump)" -pH rp_noplug.h5 > rpn.head 2>&1 +notwant "h5repack-noplugin-drops-filter-silently" "32024" rpn.head +# and stripping a filtered file loses the dataset outright, also at exit 0 +"$(h5 h5repack)" -f NONE rp.h5 rp_lost.h5 > lost.log 2>&1 +want "h5repack-noplugin-warns-on-read" "filter is not available" lost.log +"$(h5 h5ls)" rp_lost.h5 > lost.ls 2>&1 +notwant "h5repack-noplugin-loses-dataset" "ds" lost.ls + +# A cd_values array this filter did not write. h5repack turns the refusal into an unfiltered copy. +export HDF5_PLUGIN_PATH=$PLUGIN_PATH +"$(h5 h5repack)" -f "UD=32024,0,9,3,0,3,3341,20,0,1062232653,3539053052,0" plain.h5 rp_old.h5 > old.log 2>&1 +"$(h5 h5dump)" -pH rp_old.h5 > old.head 2>&1 +notwant "foreign-cdvalues-refused" "32024" old.head +./b/read rp_old.h5 > old.read 2>&1 +want "foreign-cdvalues-leaves-data-intact" "READ OK" old.read +# A data version one patch release newer is refused the same way +"$(h5 h5repack)" -f "UD=32024,0,9,$((CD_VER + 256)),32,0,16777216,4054449152,1348619730,41023,256,0" plain.h5 \ + rp_new.h5 > new.log 2>&1 +"$(h5 h5dump)" -pH rp_new.h5 > new.head 2>&1 +notwant "newer-data-version-refused" "32024" new.head +# cd_values as 3.3.0 wrote them: refused, naming their version +"$(h5 h5repack)" --enable-error-stack \ + -f "UD=32024,0,11,4081251088,50528256,136316419,8388610,0,4227858688,1305670057,2688503906,16777216,1536,768" \ + plain.h5 rp_330.h5 > v330.log 2>&1 +want "older-cdvalues-name-their-version" "data format v3.3.0" v330.log + +export HDF5_PLUGIN_PATH=$NOPLUGIN_PATH +"$(h5 h5dump)" -pH rp.h5 > d_meta.head 2>&1 +want "h5dump-header-needs-no-plugin" "FILTER_ID 32024" d_meta.head +want "h5dump-header-shows-version" "H5Z-SZ3-" d_meta.head +"$(h5 h5ls)" -v rp.h5 > d_meta.ls 2>&1 +want "h5ls-verbose-shows-version" "H5Z-SZ3-" d_meta.ls +# "ds" rather than "/ds": an argument that looks like an absolute POSIX path is rewritten into a +# Windows one by the MSYS2 runtime before a native h5dump ever sees it, and the dataset it then +# looks for is /ds. HDF5 resolves an unrooted name against the root group regardless. +if "$(h5 h5dump)" -d ds rp.h5 > d_data.out 2> d_data.err; then + bad "h5dump-data-noplugin-fails" "h5dump exited 0 with the filter unreachable" +else + ok "h5dump-data-noplugin-fails" +fi +want "h5dump-data-noplugin-message" "unable to print data" d_data.err +"$(h5 h5dump)" --enable-error-stack -d ds rp.h5 > d_stack.out 2> d_stack.err +want "h5dump-names-the-filter" "is not registered" d_stack.err +want "h5dump-names-our-version" "H5Z-SZ3-" d_stack.err +export HDF5_PLUGIN_PATH=$PLUGIN_PATH +"$(h5 h5dump)" -d ds rp.h5 > d_ok.out 2> d_ok.err +want "h5dump-data-with-plugin" "DATA {" d_ok.out +notwant "h5dump-data-with-plugin-clean" "unable to print data" d_ok.err +else +skip_all "hdf5sz3 was installed as an archive, so there is no plugin to load" \ + h5repack-applies-sz3 h5repack-records-version h5repack-output-reads-back previous-data-version-reads-back \ + h5repack-none-strips-sz3 h5repack-none-keeps-dataset \ + h5repack-noplugin-drops-filter-silently h5repack-noplugin-warns-on-read \ + h5repack-noplugin-loses-dataset \ + foreign-cdvalues-refused foreign-cdvalues-leaves-data-intact \ + stored-cdvalues-carry-data-version newer-data-version-refused older-cdvalues-name-their-version \ + h5dump-header-needs-no-plugin h5dump-header-shows-version h5ls-verbose-shows-version \ + h5dump-data-noplugin-fails h5dump-data-noplugin-message \ + h5dump-names-the-filter h5dump-names-our-version \ + h5dump-data-with-plugin h5dump-data-with-plugin-clean wide-chunk-still-compresses +fi + +# ---------------------------------------------------------------- 3. application shapes +# (a) links and registers the filter itself, nothing on the plugin path +export HDF5_PLUGIN_PATH=$NOPLUGIN_PATH +./b/app register a_init.h5 > a_init.log 2>&1 +want "app-register-registers" "REGISTER OK" a_init.log +want "app-register-writes" "WRITE OK" a_init.log +./b/read a_init.h5 > /dev/null 2>&1 && bad "app-register-file-needs-filter" "read without the filter succeeded" \ + || ok "app-register-file-needs-filter" +# (b) links but never registers it, reaching the filter only through HDF5_PLUGIN_PATH +if [ "$HAVE_PLUGIN" = 1 ]; then +export HDF5_PLUGIN_PATH=$PLUGIN_PATH +./b/app plugin a_plug.h5 > a_plug.log 2>&1 +want "app-plugin-only-writes" "WRITE OK" a_plug.log +want "app-plugin-only-avail" "AVAIL 1" a_plug.log +# (c) the plugin-only reader, the mode every third-party tool uses +./b/read a_plug.h5 > r_plug.log 2>&1 +want "plugin-only-reader-works" "READ OK" r_plug.log +export HDF5_PLUGIN_PATH=$NOPLUGIN_PATH +./b/read a_plug.h5 > r_noplug.log 2>&1 +want "plugin-only-reader-fails-without" "READ FAILED" r_noplug.log +# (d) points HDF5 at its own plugin directory with H5PLprepend, and sets no environment +./b/app prepend a_prep.h5 "$PLUGIN_PATH" > a_prep.log 2>&1 +want "app-prepend-adds-the-path" "PREPEND OK" a_prep.log +want "app-prepend-writes" "WRITE OK" a_prep.log +else +skip_all "hdf5sz3 was installed as an archive, so there is no plugin to load" \ + app-plugin-only-writes app-plugin-only-avail \ + plugin-only-reader-works plugin-only-reader-fails-without app-prepend-adds-the-path app-prepend-writes +fi +# (e) a C program, through H5Pset_sz3 +export HDF5_PLUGIN_PATH=$NOPLUGIN_PATH +./b/capi c_api.h5 > c_api.log 2>&1 +want "c-api-refuses-an-unknown-mode" "UNKNOWN MODE REFUSED" c_api.log +want "c-api-refuses-the-removed-biomdxtc" "BIOMDXTC REFUSED" c_api.log +want "c-api-applies-its-bound" "BOUND OK" c_api.log +for t in compound array int24 float128 byteswapped int24in32 floatshort; do + want "c-api-refuses-$t" "$t REFUSED" c_api.log +done +# Toolchains differ on dropping unreferenced libraries, so noref decides whether this can assert. +if [ "$HAVE_SHARED" != 1 ]; then + skipped "app-keeps-dt-needed (hdf5sz3 was installed as an archive, which the loader never records)" +elif ! command -v readelf > /dev/null 2>&1; then + skipped "app-keeps-dt-needed (no readelf)" +elif [ ! -f ./b/noref ] || [ ! -f ./b/app ]; then + bad "app-keeps-dt-needed" "one of the two probes is missing, so there is nothing to compare" +elif ! readelf -d ./b/noref > /dev/null 2>&1; then + skipped "app-keeps-dt-needed (readelf does not read this platform's executables)" +else + n_app=$(readelf -d ./b/app | grep -c 'NEEDED.*hdf5sz3') + n_ref=$(readelf -d ./b/noref | grep -c 'NEEDED.*hdf5sz3') + if [ "$n_ref" -gt 0 ]; then + skipped "app-keeps-dt-needed (this toolchain keeps unreferenced libraries too)" + elif [ "$n_app" -gt 0 ]; then + ok "app-keeps-dt-needed" + else + bad "app-keeps-dt-needed" "calling into hdf5sz3 did not keep libhdf5sz3 as a DT_NEEDED" + fi +fi + +echo +echo " $pass passed, $fail failed, $skip skipped" + +# Raise this with the check it comes with. A guard that skips the wrong list, or a section that +# stops early, otherwise shows only as a smaller number at the bottom that nobody compares. +EXPECTED=44 +ran=$((pass + fail + skip)) +if [ "$ran" -ne "$EXPECTED" ]; then + echo " the suite accounted for $ran checks, not $EXPECTED" + exit 1 +fi +exit $((fail > 0)) diff --git a/tools/H5Z-SZ3/test/runFilterSuite.cmake b/tools/H5Z-SZ3/test/runFilterSuite.cmake new file mode 100644 index 00000000..fc82c13e --- /dev/null +++ b/tools/H5Z-SZ3/test/runFilterSuite.cmake @@ -0,0 +1,12 @@ +# The suite tests an install tree, not a build tree. +file(REMOVE_RECURSE "${SZ3_PREFIX}") +execute_process( + COMMAND ${CMAKE_COMMAND} --install "${SZ3_BUILD_DIR}" --prefix "${SZ3_PREFIX}" + RESULT_VARIABLE install_result OUTPUT_QUIET) +if (NOT install_result EQUAL 0) + message(FATAL_ERROR "could not install into ${SZ3_PREFIX}") +endif () +execute_process(COMMAND bash "${SZ3_SCRIPT}" "${SZ3_PREFIX}" "${SZ3_H5BIN}" RESULT_VARIABLE suite_result) +if (NOT suite_result EQUAL 0) + message(FATAL_ERROR "filterSuite.sh reported failures") +endif () diff --git a/tools/H5Z-SZ3/test/sz3_data_3.3.2.h5 b/tools/H5Z-SZ3/test/sz3_data_3.3.2.h5 new file mode 100644 index 0000000000000000000000000000000000000000..8fc74e9d2e8698fd2d9ea491f5b7cf009f4bbc34 GIT binary patch literal 16337 zcmeIYbx@tl(?57{=ODo)xVyV^aF-y#o#5{7?(XgooCFB&F2UX1oxtXMZ*JB5zO}#I z-|qgocTUxGPj^3kWf3g2`?|<5~;NOye#{cN` z|1e;GXxJZG2L3k#2Kdhp0KonuNB!gff7+=ki;Dw(TmIw!ANuDpfH~yf9PB@Wf9e0P z5ReoXR{YO`HU6ynpItiiXP^B8hOy({+&}wK@*ln{%iko{c#8ep!&-b82H}; zp+A2AR}|v+&i{`kD$&Q^XX`(W2!I6(!1Dcz2LPY|uz%^nA^vuV0Lc4m4EE>$u)WQH zTfze**|q3Zv{>m`8Q2(@h^UMWoD7IeZ5^x)oQPN%SQ%KTd5IiNOo+^#oa`KV7#Ynh zoXnjK8H{YL867=-_jV2@j*hktjK5tlIysn_Fj^ZpI+-{yI@#JY)C5@Gp(fZ}zWcqW!k`Py5d@ zZe-dlRpb@K{&N%jEd>CGApTPS+xvR9u{d0c) z=TkX&e@lW9|E5I$)8NmTpYm@8_{Wave__|d`rj4**a`g+|GC!xyV1X+e-Zc>fqxPB z7lD5f_!ohH5%?E@|IY}Z+=$!&e#iDff8xJxtiN4SH8K9u@5W5l1}vxf5%ZuBIhkNX zaBQm^UwB~Nd|L%s>Vc{6!LNT=#y}BKEO3-`u&GlF`3hsA%C*dIL~WMDs@4$f{Jz8j zg$oV|-UX)DDu(qu z^}&oSjTWn@fe8&9yd9`W9si!blto5VqI=QM>a{vt^)IX%X?18mV2}L{mn-_!&6cF0 zeDh^KuXE_HW#R5K*x%eUevw=r|&;mpyPF@uz&BAt(HDc;Br< z-FLh{wR}dNIBRbB%&bqN{L_a8j5l3sS$cg) zoFc&G9!O}4U>9SLFSN91$f}QjkAO)HG6?7+#k3Ct66`O>86fKAp%?&&yqp7Z+Vv|d z)$5)a`p=;hfyQLOICfilJE){mDe0koi9AHY8uhl46=`$Wqz@>#5IwwK@E2&JQq;@0 zl!b-F0%ydzD{)C=zvN?loZ$}z47YX~SBvEEX(0inB*DhLNMvp^YPcSCs{+w5lA@`= zO@o;S#y!dw)S>&QL+)rnJ03&BVw%{hpVg5}(H2C9D(d;n5`Mz?tN-j3EFSqleMG}6 z9ys9vTcU<34Eciv#HpkXO)HWN_jXo|)vlOS+;(ko7XT@oULQZaY&aO69GmU{1!>+a z%}K0a=;N8x8caYJEAol5>_2tR;NzxTQbweD@2`6rcmT< z+f(Fk*^{;c*zOxFP;^u-H(tTMdn2Ua6?yMaj*c)=vyGZo{8GlzqgBSBoTb>Uia~>j zN~BtgtW5(m4-o=i5e%zgnL;Bxo)@EtRR%v-QCg0vob8tcNei!9de?};XJ^5zWT{WE zI56K>tP1@jG7|06G|D#GvSC=q86{w*g zO=cbBMf#KC^cJnt{S=$@6R$7YEws>9$^B@1!!_kdbu!Jv(weRGqL9|XubW@_@974k zk-9+97~1t1L{XvF>)Wj_33{HTY>nxD0G)u3O+VwUHjb z0!u14He^-Bv5dl^o5P@zEhC>}Oco0RT4F6gs|8j#`v>s+&e4w{=KVd2IqHoH-zaYQ|xTbGB;S>c4B_n=H=yHHiV9jE?uG1h>7J0%O`JUXgBC-TKHxkxok z`||!F^KC|v@=3NK_S+Gb>%Eie;j-82hwnqw z+k>b)+t<@g7x^l+u@%hoxjQdy9c(~yqnXE2t(j3#EE#c%Hvt51hw{77O}=CFmnofR zIi3tXESSYZB!gdFl$J^d2gzP~lV%pfcN3DbHNKsiTEEa^{d^%B9u;ekCYdJ%ESeEL zOFzcCf;!b6i{gq(AL zgtpl}nMnv*Z~J+ADCPTm55d27XG=lQEkyvBaqn&F4gA zFe2LAGzL{avV{lw`AK$#q33>(=F{Tgdh{UL7?pLOj?5!p88f;81LI@C6*EP{i)oN@dh z)E5yk>eb$a!hi*szA6O&jv8?U*LSsx^G?|xp)_t(p=61_z>4zMdfrZp?x zKd{|9(ms~Ci%9InzB{_?0a0PvI?tp0LnPP>;s4)~H_b+TjC zr3jze*!Q8~1&C?W)|CpQ8H26~`%|BP3Hy-NaF+@s!chxGNaxiWL6+VPlR3TEDe|fS zok<7x@*Pr6U;=<;EMTaUaZ-uXNo2`0cGqxBZetaej%ayTMswJuV&oPTB9$GCO%=r1Za1ybbbi3`p@vU6eAZf;3x~-7X*13w5P$p(6K$G{SEIekR!}9 zj_%%hcdl-8@T=_oJ!7;;Ipk7+X>EB(XfGR_1o=fNz2q7qxu}w!2rEXzCN;C}kk!1z=5GWowh!kiE_(jiU-|FxoKhG zL%w4Yi!L2Y^sVI-^Y~mR0}r$dBRno^9L-mq7?(@R--TTBPthyo646)slf)uB1?aZ4-Hs0>RZ2dOuP? zO2?x$m?gsx)n4|!IlIKoQlM9wB2*tE@3U@#9;ZcLoYu1LPY!Xf0AUMPP` zTK{LIF`_bsAq`Vb^ELV*bn^2%VeuIWoZvC{(;F_k#4;TI*ykJ;{EcKLiBj;`!n4Ph z*3$ISZcw|mIV)y6zBQ$w!b?UwlFQ6WNJiMDL)r4?LqdP0H(H!J@CLsDUw5P~d>mDn z(x4+uj{DH2EQxhk)oeC}1lRv%%l$(~(vGzwu95`gDhS>@)I^2|T#w>($!3a%kWb~> zqztPbb&lMGK^tiqK0s+p*07G)UiBxDFQo&%M-C9*Sgqke*f$2JbO8-Cm4E2mnES?c zW_!)DEM<#lNg)n|c{y6Juuc5bkZjkxNI|4DQOQchW_jh23sr;>>W;h=n#y91eu_fM z`o^V$xU31k)^|q&t-0T}Yj4B*%4LJxliRbEQcfXxOjLzg7k-mtLa+h9NX528mg?eEXLa^}qIT;)Q9IgyjoQ&L*R%j0nM1_9D8Fm}*bp=`tmhst zuRtL7*rNT7e>>*FPn4BXmUOta!*j|8@d=Ai!d>UfGM~;^B|M8!#0vQz*6HBz;6U(c zu$=ST-K>F4Iiu$QdVUK@?aM+Ox$X!7r+hemb77gw9hB29ADi|`FPlxijr0je-Om&D zAq|-eRi;n^4e6tF`;4A#qn~}=LV!6YbO`IK7o86SDoGVfyKJ?sszwxeoJl^!7K{2V zX>vSpB-D?t0)wwnHMXeEsV2OP!wwF-sa8>KqEW5fuBNp-EHQjWFcYA{@=@C|lQUA4 z856rCCM?+kRyskoznAH5Y%^~THIVHH6OoNQURGKTB0}!?Fg#rVpe#-x%2m?5WtYNJ z&s?BfS18?toukwb{Z#Cg(ptt6C6i_A)lJUJ&RJ}S5DxxVPstC}JrN;UoamO7} z9Hk7-?JOHjV&Xca%Je1~#B5}h7*yZO%V?pN$WLQHyez0|izAS7iqa^h^nu3+C-Guq z3r7XEa0)J!rn*@22WAx+=)-A&XfTHZsx>JbAtyM@>uD}jU#b7RW9YOZt>(j9B8(N=31ik^h+R( zgtQ()TLM=fAB@>EFg1eHz&Bx54~fJWHPo~EX7)e`qdA>j?^BFa=}>h z3OWVnQ*yg0u3UH*We25XSH9rG{`FcgLVn5w{b<_$)7?s0<3!39ukK>?j z8!fp^3uYZ53rgW<@5UG!#l~<%)Ma5>b7BbLIv^K^O|II*t|p(${^|C}5*LCQmU*Em z#&p^eiws3ns`##9Yw&jyxF*;d^g7#GI=%I^<#A>#n@YOmY!dPj5zR04vMy3+q?zhs zL(rN;FNL8o>CI*;<~ld&+bbr-eNnq!VNNp4?F{XeA6$?dcU?32 zMO0ks5yZzk`1i@#QbyGcSV08z9R=pdlPf>2UBLTUf_CAg@=J%3 zi@O!sCK4T? z-K4qhx&6_@xVYYht66$W}wL_cga z8BDN_FMnHi4?aFZHXwe}FBDRMb;sIU(HYjhn<@oAY$9m_z0RPnukfOuK!2RDpWzfW zx%B^R?{TppG&uyJoUZN5kdVG}_Pp!-+{H9D04yqOu?o_|HiCNyGICQ<7z$a>8z9Jr?sFqMPoXvwf(KelVZWy+c8<9k~sl|@rx61mavc{s?L@6C`_dNX>MRF}w4fyTMzNvsPP_7I7~&($}wUbpwZEsA+7L zR+o>W`{Cv=sRVqmVa)2{)LSJL(Z;(_6x3CnJmtbAPjUnPV!gfaXwu;GYDL7MWzDNo zG-NJqq%W$yqS@uR=6V)#6Xzeol?T*8N~4wF8Oq8s#E*Fu(_p|6JXD90vK*`ow{rah zMqwK2XAByWJwwF=Xi8=xEVyYTi#AhGSQQ)zbsGt><~CF=3tdl;4yrMPWIwTpmKBPV zq#PjYGk2KQLbA*h4a8iA(RgKJO=Vgq9}%JVbb4Gsad%EXPAX_V-QaU#CLN?fGgOe~ zC*-f@5_{*oW87PO`7dQ6vSq#Q2C#h`LpdsHHwLmNhwpy!uUghKn>Q&XhJ&<$~fU- zATpGAGIDCkglKj-vsNyZdw@P<=D#CsivAKZOqlJ9t+Ip z?)v-`xVrwul*rh|pd3y&=dQ-NewV0T zGM}6-(JEO!ayY4={QW$CKXqDHdIK!ZA+z&6Md|qnQPQPa0-uiX`LflBfK=z@*Al8a z_`|sQ*r=QLh0aFwK)X41oWXl()!5Nb2R81PLit46W9qCs*Y;~-plX+h0lHudICPnk zX9r*oqDpXnhp{fxv;#`wMLF!JK?}>H?tVF$5g6}{fhB9&3Az(!`BB@5y9 zwKd+Ze~S7{ij=ry{)6#+A}3?-gTu-AZ{G?EyPZ+r91lz$J5SxE!2DCK=%DbaPYAfN zJQ+hT_Hss*crQLXxV^f0a$6o`k8XLk9XQNQ@TTG;V?D^%@I381JnEje;Wyn(kcW;& zccd1&theEJre>-hCA}Z#o!iH-=yAN4H}Cvhe5-14Je%HaUlverzTmRFS+rd>FnnoP zk8w2l-A^=0?2P*Cbd(;&HYdp%y*P@K&nMs48|JLCa{up#=PG|bf)*=gr4o(Yx1jbkqxoij(;n4#b&QoCHf2hc945i4dTIG?itA>DHQ2|K< zqgKJ8uh=qSF>ARj8##lV?GdALJ1Bf_8BLq$9wM}A zz7B`OufI+M)1^nN8nVK>PUVO452YW7_ZBC!xwDx+ouE|gim9`oipUANQC7`CjLQz< zf=mTSI_ASLNajN?r*06eb+-d|?Om?AUK0*6j^HinsV$)V~i28WY3d3<>UkjdA2_UgYXz zjy6Nw!So&9{@}Uh9NzM~urIu7P~CvMiJ)h_1^0>Gpn`zTIj}G!qg!FY74$=}N(S$7 zCW$A)o_OzOK=E1Q6P0&gUy^|QWKnEf>dYW>SrYI=Kc14jAt&Y_K&oy3v?vy6EKSbF zlqeu=X!#j37@Rs+NHE$o!8nd3FTGd^CO;<$6@p*~3m#4qE6p%z;%HXl7F2&^g7P-h`~W*$$9stNXAFA;uQ}R@1Mw$f0Z2oj`JG$*d&|8q z@pk(g5Rou8wDMKdi?1Kl^Dk?VvnM;mIV7}JKhYH~#U zvKCxxt5VfcuY-kLx~VX-(%}v+vH3Vethz?6dAVpkKYgA^V2CF6SroA779>-5mf%12lA`9D^G;-?3VvBS1c_fykC^8 z&;F5J3zmVp#etbKpt=ien4S#Q0ql%O0MQD<^U^BEWL1L4!9^b2`sX-J33r1zB9;E| zDOjZ`)`McTzDBEFE8nM4mp1TLe{a?1k>|6`>@{h3k`_mV;_wv)1ML;kOyg|2oH!!S z`@q3_4I(3h$ik4Q?5`XggAI#}A9PNds6jy*s058Jy&f89D~6$;l@{UE5DZA^06&m@ z@LomEg8ADMq4fNQXHqAMsJqdH9BvCqJKUw*O-Nt_Zpre65UzU<2H)|y_X+t9$_u-k znhOv2Im5$B-N)^IPn88iC{|v`AQj5Zkb6|??HY{rS#2p|ea-D+e)<<=&9Hn7uXF+_ zDf2Zt_gR%y10gTP%Ah$djujsB=T}1<8u=*08q~zaK3YCLm@4Kfe{g`daN2V>YRzS6 zugkf5?>mXmSpm6BzcLI&_I&M#?tG;?hJv(CFbmA5jfzBaY0Cssi+M?U7+T!$S(CT7ud#dFWjA*ie02a zknnn@uYA7DYlm>-kD#pEBw74lv%|X|L*idb!DTS{Z$;Kqd^5VQwq?xWO@FB%M~7-< z-2Z|aEnAeSA1Qn;3+t*6MJ9O>m7vj{Or^f)l_({7G*4O0^L;)O=U-D>FU~z-=6VdY za3FlMV^zG&5AgT72-4pa9In!IFEwo(rsc6;I3yl*%J!y^3ow26620?JT<6HGt)2Kt zq4iiAAN`vBSvNdXImV5~okj56;$aHJg{2^jD@gTTHtBfvv?(BM5;+<=Go*bz$>YxO zE0=Px)J%Wy_MK`W2Z?@MFyA&@B_|@~duQk@)0)8057H`YM1f!vF!Rfh2 zv#hds+gr2KTqfx6K5^IdS8&wdU$ybe?$Tbfb{#X{OU#ga(0Py#?y9b9UR6;2ePi4e z&W;vSh@XOz06T;GHg~ z;r+va&W@g)5`vtQWn(*PPh4dObw4c5n_;#{C$vYdD!oQODiH#ufK%^JSF^XFB_4d$ za9x*)$vnTdw_%j8BI5KU`%m1jeXwcWBq|^{_zKu0fkPx%0rk|&QVulS+43D+PvRN! zCsTG@Y|ulf11wm^TJVL(6hq%XTuZ{q#Z+Uvx>-8K;qh`W0k}l9=DfXlZRhBPRv9 z)P>58oR_{U8c$895}J`9$*T-uDSH`ghGPPGmN)vjYyhHHf)cbiF{#D^MUTnzPE%M3tUq!lnRG0Snjh_9 zpPV_XkzxjJ5u%8ZzqL7;qL@-il$4m1RFDE0@uImW+E-8lGjjw?PYcL54f=*ps856r zZXtV|QTGc2D)ZowTeo9(k&qVN2Z(&(?_Kj|3;7a|bV5lH8CgZ~&D@UPd;EH;Vu}P^ zI-w~^hgenty@=D0l!8RU335nV%-Qlw9CH0m86@(hv-~m~a;39Fl!bE8k|+jn%|rdr zkYAyr46$RrqKgboB%nm9h$pAaBLRUtunExAnY${G3YG`)TDV~}CK+UQ2Hj<&+hs%a zAvdF{L?jv(-kTbEQ$t6VmX(RSg+wxhkd<9UT(BwPLXq1+OHJVC6X=-)F=Nn`6JABI zp3uK5`Y*8j??E$_R~{NlZ^Nb&g5Aafn6<15Wrw++j5^m-+J`8;9QV$1NdcpLc^Dld z3z-eBPoDNUQ@^xL<__X|uBKQ`p}cnYBV4uCqy+MrU@ole%=U@>(Ai%kBsJO-du&T= z!(>&5cxSFZJ-AKvgq}L&V>Oz^-oDg{dx(5?Ad!DE>W_hXC&0><7yL&!8#L4*od zMWPJ99THH#qa-}#2N#g0(G3HoE-F36S6KnV^0S`}uikaRB|QC3t`$5zR{)_vptL0e zTKrz=n>8NeU)|?*JKqY%uqHasn5kRqGcP8gCLQhj?E*gVEao*Aa{N8n@$J$Y+27`T z0T$iifE+!Q11p0FO0hgX0e|i1hWcBMVb*O%ANUt2O z+)dRlWLl<00$mj;-R_NR((`nX>ezydg1QW9GM3X$yipD+PbblHPn>p_VUe)T`+at^ zC;@a>q*3xN%&IEaR+<9$9fHj4nuKxRYJm*80?(!$n%fvw0V(4*OoR7)F;||i4QN5i zzf9+n9LC=ZzE2Ykb`7sueB&#Fe~}7MIU|;3mmXHca208vE$1^^SzwmA)Mba@}AXlDo0>BAqzSPe!J z$ot{U1~Jx>QkB?bef-jDl+Sw=x_1x;$#{JWof~~BIpAU`Q8L>4{HiqSMvJx!P9E5D ziifV<#?dh1W*FP2+R%x#NZfU5CEU7_clRyK`3gQk{l@zv57syGcZNNDiXCxj+xzh` zt2AeK{&E);wh)-{kP`-kb5eD80{4`?TdA+y+N3V#`UE^@9J3*P*@&GExgm6sS~xlx zd={)W5}Rp=6ZNqU0NhdH*^E_Y43B3XWxu!4v&>|K$%IW$V}6BjbndfnDfbqO5wlBq z<}je0op@AMi|RO}(fo2FK*$6CJ<*5J8JXgNLXPul9FZfdqftQfiwfROAtdEc;!>DX z6M;OL{%EudLHv~=Bm?VUzMN{TYAuk-Bu>R3HRMr}e%1-60D|6e`2o?7rLNfMoArf9|-9;7kl!3rL*ago=YjmBM0X_4iBKM7g;^Nt`G=f5N&!Pq*wfQSM*QehM(1@dUH zD?Em>IUESW7-$M^&ud~FikK~$`zJ;<^yg}|CbVVQrL`Mhm~6*@t^@pA2hFxb%zmtI zUkVRqf2ShI;1u9HVEBMm#r%rkXVq-W;BT4tkaA?Fun0Kdu-X>NMQ8fNXQUv!D(DDg zdgw3aD0v-S0sJSe9vqYE`k@mO>41q&P0lYVU=`9|hlOcLue!LVpfO9HWicfB zIFp>ySj5jY6Ki@~z^*YJ4r;iCICX=1&pwki<5sZCBFCPvb#crX|GdvE z%9Qrw)MTLPv@4Q!qAUEPjoVfL407wu{#fcm$3xPMyid!T^C-bX;)}?7Rj2A4Ykk`^JegK;&@QmAHY&!|4GGazg>i24ryTRWo4}Ja@uaZLfLS z^__V-6#n@JNw>noC?vwutJe8eMacQKIjzD6ROI=i0+Li&=&(XaQhfQA1KWypLHN_m zGMwgEA`Dc85HVCZ^vq@3ICM_Yxy*-oJ4kFXDVWQpM@Z5K=WzslAVppeqTr=rBZ{W* zi|;aY^s-ar%<=Ol46+vrhP8#@rDXS0>(UFk!p}5Zgt5n&&{;>B;WM<12jd^!q*1;C z>3C1l9p0oL_cH|fE{vI*kFuSTgmsU46#Pdhp=xO=36t8SANJL=Ra8S z6CnF&z(u-f;&9c&RoNRWJ*?NtxM;ADa4<%pB9{v~%QPt@=g`0*6)TSc$^;cKNZ6LY zvk&=sbY}_gpK?QuwEHL2Yw_wep5& z2eL&j8L&*Y%963PX#H_3JjSTee*gaQdMz~If^_~Sa?KV?ogSt_@n(y!r?R>&Mr&Zb z((XE@%>hSSO0%W(7{_n{j+FlIx_!L?ZL>s5HOx0&2P^so7Yrez2E*1xELtP$BBrmD zAlNzxIye+%UW{S`PODI)uQWeHRdnDpWx1{oU`e9jAffzXB1`epaoKR3S$rO+C+fzcv-Hi;WK3L;%M z;qZYUqlB{E^HAI`@pN1nwv&(&^$f>Zl9W&ITZDzhu_6kUGj8f=wuv&Tv7j@e!=kgf zifoXEMd?xl63rU`ahbBVkQBVP3EtOA-n=(-vlCLQT<2@#YXOh1ApS^7T3Z}WQfFn3Djjyb-9_HpJ|Eu*x3axp2mIxl+Ga)7hR0+&`r zol`lC3Bc<_jQi8|OY!G@1c){2=fNJ}{h?zRU@Kek6Ax2PY>q;&+~h%`{BDEH?9Ip&QNuTb6Y&Tt5KgOiba$SgfB}D> z*j&0lJCzk8DK49tv$|rmK-J;m>Mu#UB%B5oIrJ}z-GunMLKbyxx<L$z9ccPF0}mhu18ofHg;+|)VP9FlpF;C*-4>EvlT2HbX*~ye5sj|9&q~Z zYINOT+oV>$G(h?wf<}^*e*r`Onzi3HIF^_+N0R-HITX%5Y4S=G4<2D85lE?MGx`Hc zsc|-ZLY0(7F9>BbMY0T$g%|q+QN>kjWP^A=S~H?B)BN>v-!}4ioB42rII|^@2c2kF z1w$KbfM^|;q!25aA111nRX{)H-R^BE;B$DBh>P{T*ECpP&qt6&CTy|MU@0(~RjZM& zPFO14#^c`oI{DH|z?UyKKF5h9SB-;TypEJIkdgt^LL(|yG{o=Sw-PUF2)U_R1eS=am6H#_F?V;x3Lm)>#{;Tlk4>)&DNO&ioh4^;N28iq+)Y z&4CvsIwfWfj?wULCWNea!a3bW6hI=}EP?)nkCptz6#g5+4>E(xbT>&ilft!^d?}h- z>0wbX(a7-0MzjE`vg<`M6lN0+QmF)07i2@k4}>HnXq;&JB7tbcp<<@L-%3DHQ&OMe z6IbQMFf}1vLHDXh#5E5}5q*F+kf|ZvEzqPgV^e6H?jc&B$cUmlbvX>9Uz$7}s!sqB z_m~$Ij_Op!A6GRTKR}9!C@}H{+jRg9NndgalycO} ztB`gKgDL;hrCExs$evzm*!t+49;3FALjA&fw#Tw}(qR`0w>9>>87#Yt?ehM9u=wp0 z`npk{iq)(1TLQRO`Xpid%guhZhn) zrQ!O_r>+87ROG#W^j^k|q;gB*G3*R|VR2uRGW-P_!5H*S<*M?t-4fs!YwOW)^Ju8| zB(){%jt3gm0bh*ySK;>>V7~BphpMp0g@}C!HM>odgJas!+gkwt2tZSG9QzRXKj`3Y~jb_P9P}rOqCS+B3 zW$?Ek#8WjXwc`1lvL@ z!0JbW9^hPv5p(<2V9WS-`wg2>{G&=3T`SW0_@sE{E1ug2$!%)*DJ9Zx9d{hPJMArw zx6?D;2UD-Dbc?&AODj4cdeM`P{ePbh F{2wG`a5n$| literal 0 HcmV?d00001 diff --git a/tools/H5Z-SZ3/tools/CMakeLists.txt b/tools/H5Z-SZ3/tools/CMakeLists.txt new file mode 100644 index 00000000..870c623b --- /dev/null +++ b/tools/H5Z-SZ3/tools/CMakeLists.txt @@ -0,0 +1,10 @@ +function(build_hdf5_tool) + get_filename_component(tool_name ${ARGV0} NAME_WE) + add_executable(${tool_name} ${ARGV}) + target_link_libraries(${tool_name} PUBLIC SZ3 hdf5sz3) +endfunction(build_hdf5_tool) + +build_hdf5_tool(sz3ToHDF5.cpp) +build_hdf5_tool(dsz3FromHDF5.cpp) +build_hdf5_tool(convertBinToHDF5.cpp) +build_hdf5_tool(cdvalueHelper.cpp) diff --git a/tools/H5Z-SZ3/tools/cdvalueHelper.cpp b/tools/H5Z-SZ3/tools/cdvalueHelper.cpp new file mode 100644 index 00000000..466a8d7e --- /dev/null +++ b/tools/H5Z-SZ3/tools/cdvalueHelper.cpp @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/compressor/Compressor.hpp" +#include "SZ3/utils/ByteUtil.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/FileUtil.hpp" + +inline void usage() { + printf("Usage: print_h5repack_args \n"); + printf("Options:\n"); + printf("\t-c config_file_path\n\t\t(read config file and print h5repack arguments to stdout)\n"); + printf("\t-r 'cd_values_string' -o output_config_file_path\n\t\t(read h5repack arguments and write config file)\n"); + printf("* examples: \n"); + printf( + "\tprint_h5repack_args -c sz3.conf\n" + "\t (output: UD=32024,0,11,4077060608,16843009,0,33554432,4054449152,1348619730,16818239,257,2147483904,2147483648,16777216)\n"); + printf( + "\tprint_h5repack_args -r '32024,0,11,4077060608,16843009,0,33554432,4054449152,1348619730,16818239,257,2147483904,2147483648,16777216' -o sz3.conf\n"); + exit(0); +} + +void parse_cd_values(const std::string& s, std::vector& values) { + std::stringstream ss(s); + std::string item; + // The format is filter_id,flags,nelmts,val1,val2,... + // The config data is in val1, val2, ... + // So we skip the first 3 items. + std::getline(ss, item, ','); // filter id + std::getline(ss, item, ','); // flags + std::getline(ss, item, ','); // nelmts + while (std::getline(ss, item, ',')) { + values.push_back(std::stoul(item)); + } +} + + +int main(int argc, char* argv[]) { + if (argc == 1) { + usage(); + } + char* conPath = nullptr; + char* cdValuesStr = nullptr; + char* outPath = nullptr; + bool reverse_mode = false; + + for (int i = 1; i < argc; i++) { + if (argv[i][0] != '-') { + usage(); + } + switch (argv[i][1]) { + case 'c': + if (++i == argc) usage(); + conPath = argv[i]; + break; + case 'r': + reverse_mode = true; + if (++i == argc) usage(); + cdValuesStr = argv[i]; + break; + case 'o': + if (++i == argc) usage(); + outPath = argv[i]; + break; + default: + usage(); + break; + } + } + + if (reverse_mode) { + if (!cdValuesStr || !outPath) { + usage(); + } + std::vector cd_values; + std::string cd_values_str(cdValuesStr); + + size_t pos = cd_values_str.find("UD="); + if (pos != std::string::npos) { + cd_values_str = cd_values_str.substr(pos + 3); + } + + parse_cd_values(cd_values_str, cd_values); + + SZ3::Config conf; + auto buffer = reinterpret_cast(cd_values.data()); + size_t cd_bytes = cd_values.size() * sizeof(unsigned int); + // The data version's first byte is 0; a v3.3.2 Config starts with its length, never 0. + if (cd_bytes > 0 && buffer[0] == 0) { + buffer += sizeof(unsigned int); + cd_bytes -= sizeof(unsigned int); + } + conf.load(buffer, cd_bytes); + + std::ofstream file(outPath); + if (!file.is_open()) { + printf("Error: cannot open output file %s\n", outPath); + return 1; + } + file << conf.save_ini(); + file.close(); + printf("Config file saved to %s\n", outPath); + } else { + if (!conPath) { + usage(); + } + SZ3::Config conf; + conf.loadcfg(conPath); + std::vector cd_values(1 + std::ceil(conf.size_est() / 1.0 / sizeof(int)), 0); + cd_values[0] = versionInt(SZ3_DATA_VER); + auto buffer = reinterpret_cast(cd_values.data() + 1); + auto confSizeReal = conf.save(buffer); + + int cd_nelmts = 1 + std::ceil(confSizeReal / 1.0 / sizeof(int)); + // conf.print(); + if (cd_nelmts >= 20) { + printf("h5repack can only take 20 cd_values, but got %d\n cd_values in SZ3", cd_nelmts); + return 0; + } + printf("UD=32024,0,%d", cd_nelmts); + for (int i = 0; i < cd_nelmts; i++) { +#if SZ3_BIG_ENDIAN + printf(",%u", SZ3::byteswap(cd_values[i])); +#else + printf(",%u", cd_values[i]); +#endif + } + printf("\n"); + } + + return 0; +} \ No newline at end of file diff --git a/tools/H5Z-SZ3/tools/cdvalueHelper.py b/tools/H5Z-SZ3/tools/cdvalueHelper.py new file mode 100644 index 00000000..43a5712f --- /dev/null +++ b/tools/H5Z-SZ3/tools/cdvalueHelper.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +""" +Convert SZ3 parameters to HDF5 cd_values +Note: many configuration parameters are set to placeholder values + and will be overwritten when the HDF5 filter calls the set_local function. +""" + +import sys +import struct + +DATA_VERSION = (3, 4, 0) + +class SZ3: + def __init__(self, algo="ALGO_INTERP_LORENZO", absolute=None, relative=None, psnr=None, l2norm=None): + if sum(x is not None for x in [absolute, relative, psnr, l2norm]) != 1: + raise ValueError("Specify exactly one error bound: absolute, relative, psnr, or l2norm.") + + algo_map = { + "ALGO_LORENZO_REG": 0, + "ALGO_INTERP_LORENZO": 1, + "ALGO_INTERP": 2, + "ALGO_NOPRED": 3, + "ALGO_LOSSLESS": 4, + "ALGO_BIOMD": 5, + } + if algo == "ALGO_BIOMDXTC": + raise ValueError("ALGO_BIOMDXTC was removed in SZ3 3.4.0; use ALGO_BIOMD") + algo_val = algo_map.get(algo) + if algo_val is None: + raise ValueError(f"Unknown compression algorithm: {algo}") + + # placeholder values will be overwritten when the HDF5 filter calls the set_local function, + serialized = bytearray() + + serialized.extend(struct.pack(' +#include +#include +#include + +#include "hdf5.h" + +static int computeDimension(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { + int dimension; + if (r1 == 0) { + dimension = 0; + } else if (r2 == 0) { + dimension = 1; + } else if (r3 == 0) { + dimension = 2; + } else if (r4 == 0) { + dimension = 3; + } else if (r5 == 0) { + dimension = 4; + } else { + dimension = 5; + } + return dimension; +} + +static size_t computeDataLength(size_t r5, size_t r4, size_t r3, size_t r2, size_t r1) { + size_t dataLength; + if (r1 == 0) { + dataLength = 0; + } else if (r2 == 0) { + dataLength = r1; + } else if (r3 == 0) { + dataLength = r1 * r2; + } else if (r4 == 0) { + dataLength = r1 * r2 * r3; + } else if (r5 == 0) { + dataLength = r1 * r2 * r3 * r4; + } else { + dataLength = r1 * r2 * r3 * r4 * r5; + } + return dataLength; +} + +int main(int argc, char *argv[]) { + hid_t file_id, dataset_id, dataspace_id; /* identifiers */ + hsize_t dims[3]; + herr_t status = 0; + size_t r1 = 0, r2 = 0, r3 = 0, r4 = 0, r5 = 0; + + char varName[100], infile[100], outfile[100], database[100]; + char datatype[100]; + + if (argc < 5) { + printf("Usage: convertBinToHDF5 [datatype] [varName] [infile] [r1, r2, r3, ....]\n"); + printf("[datatype] options: -f float32\t-d float64\t-i32 int32\t-i64 int64\n"); + exit(0); + } + + strcpy(datatype, argv[1]); + strcpy(varName, argv[2]); + strcpy(infile, argv[3]); + + if (argc >= 5) r1 = atoi(argv[4]); // 8 + if (argc >= 6) r2 = atoi(argv[5]); // 8 + if (argc >= 7) r3 = atoi(argv[6]); // 128 + if (argc >= 8) r4 = atoi(argv[7]); + if (argc >= 9) r5 = atoi(argv[8]); + + int dim = computeDimension(r5, r4, r3, r2, r1); + size_t nbEle = computeDataLength(r5, r4, r3, r2, r1); + + snprintf(outfile, 100, "%s.h5", infile); + + /* Create a new file using default properties. */ + file_id = H5Fcreate(outfile, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + /* Create the data space for the dataset. */ + if (dim == 1) + dims[0] = r1; + else if (dim == 2) { + dims[0] = r2; + dims[1] = r1; + } else if (dim == 3) { + dims[0] = r3; + dims[1] = r2; + dims[2] = r1; + } else { + printf("Error: wrong dimension\n"); + exit(0); + } + dataspace_id = H5Screate_simple(dim, dims, NULL); + + /* Create the dataset. */ + snprintf(database, 100, "/%s", varName); + + if (strcmp(datatype, "-f") == 0) { + FILE *f; + f = fopen(infile, "rb"); + if (f == NULL) { + printf("Error opening file\n"); + exit(1); + } + auto *data = static_cast(malloc(nbEle * sizeof(float))); + size_t elements_read = fread(data, sizeof(float), nbEle, f); + if (elements_read != nbEle) { + printf("Error reading file\n"); + free(data); + fclose(f); + exit(1); + } + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_IEEE_F32LE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + + hsize_t di_ms[H5S_MAX_RANK]; + int ndims; + ndims = H5Sget_simple_extent_dims(dataspace_id, di_ms, NULL); + printf("NDIMS: %i\n", ndims); + free(data); + } else if (strcmp(datatype, "-d") == 0) { + FILE *f; + f = fopen(infile, "rb"); + if (f == NULL) { + printf("Error opening file\n"); + exit(1); + } + double *data = static_cast(malloc(nbEle * sizeof(double))); + size_t elements_read = fread(data, sizeof(double), nbEle, f); + if (elements_read != nbEle) { + printf("Error reading file\n"); + free(data); + fclose(f); + exit(1); + } + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_IEEE_F64LE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + free(data); + } else if (strcmp(datatype, "-i32") == 0) { + FILE *f; + f = fopen(infile, "rb"); + if (f == NULL) { + printf("Error opening file\n"); + exit(1); + } + int *data = static_cast(malloc(nbEle * sizeof(int))); + size_t elements_read = fread(data, sizeof(int), nbEle, f); + if (elements_read != nbEle) { + printf("Error reading file\n"); + free(data); + fclose(f); + exit(1); + } + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_STD_I32LE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + free(data); + } else if (strcmp(datatype, "-i64") == 0) { + FILE *f; + f = fopen(infile, "rb"); + if (f == NULL) { + printf("Error opening file\n"); + exit(1); + } + int64_t *data = static_cast(malloc(nbEle * sizeof(int64_t))); + size_t elements_read = fread(data, sizeof(int64_t), nbEle, f); + if (elements_read != nbEle) { + printf("Error reading file\n"); + free(data); + fclose(f); + exit(1); + } + fclose(f); + + dataset_id = H5Dcreate2(file_id, database, H5T_STD_I64LE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + status = H5Dwrite(dataset_id, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + free(data); + } else { + printf("Invalid datatype, use ./convertBinToHDF5 to see usage"); + exit(0); + } + + if (status < 0) { + printf("Error: H5Dread < 0\n"); + exit(0); + } + /* End access to the dataset and release resources used by it. */ + status = H5Dclose(dataset_id); + + /* Terminate access to the data space. */ + status = H5Sclose(dataspace_id); + + /* Close the file. */ + status = H5Fclose(file_id); +} diff --git a/tools/H5Z-SZ3/tools/dsz3FromHDF5.cpp b/tools/H5Z-SZ3/tools/dsz3FromHDF5.cpp new file mode 100644 index 00000000..6da67df9 --- /dev/null +++ b/tools/H5Z-SZ3/tools/dsz3FromHDF5.cpp @@ -0,0 +1,300 @@ +/** + * @file dszFromHDF5.c + * @author Sheng Di + * @date July, 2017 + * @brief This is an example of using decompression interface (HDF5) + * (C) 2017 by Mathematics and Computer Science (MCS), Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + +// #include +#include +#include +#include +#include +#include + +#include "H5Z_SZ3.hpp" +#include "hdf5.h" + +#define ERROR(FNAME) \ + do { \ + int saved_errno = errno; \ + fprintf(stderr, #FNAME " failed at line %d, errno=%d (%s)\n", __LINE__, saved_errno, \ + saved_errno ? strerror(saved_errno) : "ok"); \ + return 1; \ + } while (0) + +#define DATASET "testdata_compressed" +#define MAX_CHUNK_SIZE 4294967295 // 2^32-1 + +// H5Fcreate runs before the reads, so a failed read would leave an empty output file behind. +// The handles are closed first because Windows refuses to unlink a file that is still open. +#define READ_ERROR() \ + do { \ + H5Pclose(cpid); \ + H5Sclose(sid); \ + H5Fclose(fid); \ + if (remove(outputFilePath) != 0) \ + fprintf(stderr, "could not remove %s: %s\n", outputFilePath, strerror(errno)); \ + ERROR(H5Dread); \ + } while (0) + +int main(int argc, char *argv[]) { + // int dimSize = 0; + // size_t r5 = 0, r4 = 0, r3 = 0, r2 = 0, r1 = 0; + size_t nbEle = 0; + char hdf5FilePath[640], outputFilePath[1024]; + hid_t file, dset, dcpl, space_id, dtype; /*Handles*/ + hid_t fid, sid, cpid, idsid; /*Output Handles*/ + // H5Z_filter_t filter_id = 0; + herr_t status = 0; + H5T_class_t type_class; + H5T_sign_t dsign; + H5T_order_t dorder; + + htri_t avail; + // char filter_name[80]; + // unsigned int flags = 0; + // size_t nelmts = 0; + size_t dsize; + // unsigned int values_out[7] = {0, 0, 0, 0, 0, 0, 0}; //at most 7 parameters + + // hold dims + hsize_t dims[H5S_MAX_RANK]; + int ndims, dim; + + if (argc < 2) { + printf("Test case: dszFromHDF5 [hdf5FilePath]\n"); + printf("Example 1: dszFromHDF5 testdata/x86/testfloat_8_8_128.dat.sz3.h5\n"); + printf("Example 2: dszFromHDF5 testdata/x86/testint32_8x8x8.dat.sz3.h5\n"); + exit(0); + } + + snprintf(hdf5FilePath, sizeof(hdf5FilePath), "%s", argv[1]); + snprintf(outputFilePath, sizeof(outputFilePath), "%s.out.h5", hdf5FilePath); + + /*Open the hdf5 file with SZ-compressed data*/ + file = H5Fopen(hdf5FilePath, H5F_ACC_RDONLY, H5P_DEFAULT); + dset = H5Dopen(file, DATASET, H5P_DEFAULT); + + /*Retrieve dataset creation property list.*/ + dcpl = H5Dget_create_plist(dset); + + herr_t ret = H5Zregister(H5PLget_plugin_info()); + if (ret < 0) { + printf("Error: H5Zregister < 0\n"); + exit(0); + } + /*Check that filter is not registered with the library yet*/ + avail = H5Zfilter_avail(H5Z_FILTER_SZ3); + if (!avail) + printf("sz3 filter is not yet available after the H5Pget_filter call.\n"); + else + printf("sz3 filter is available.\n"); + + space_id = H5Dget_space(dset); + nbEle = H5Sget_simple_extent_npoints(space_id); + + if ((dtype = H5Dget_type(dset)) < 0) printf("Error: H5Dget_type(dset) < 0\n"); + + /*Read the data using the default properties.*/ + printf("....Reading SZ3 compressed data .....................\n"); + + if ((type_class = H5Tget_class(dtype)) < 0) { + printf("Error: H5Tget_class<0\n"); + exit(0); + } + if (0 == (dsize = H5Tget_size(dtype))) { + printf("Error: H5Tget_size==0\n"); + exit(0); + } + + if ((dorder = H5Tget_order(dtype)) < 0) printf("Error: H5Tget_order<0\n"); + + // create output file /*Get Dimensions for writing*/ + + if (0 > (ndims = H5Sget_simple_extent_dims(space_id, dims, nullptr))) ERROR(H5Sget_simple_extent_dims); + + int ndims_used = 0; + for (int i = 0; i < ndims; i++) { + if (dims[i] <= 1) continue; + ndims_used++; + } + + dim = ndims_used; + + /* create HDF5 file */ + if (0 > (fid = H5Fcreate(outputFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) ERROR(H5Fcreate); + + /*Create dataspace. Setting maximum size */ + if (0 > (sid = H5Screate_simple(dim, dims, NULL))) ERROR(H5Screate_simple); + + /* setup dataset creation properties */ + if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) ERROR(H5Pcreate); + + switch (type_class) { + case H5T_FLOAT: + if (H5Tequal(dtype, H5T_IEEE_F32BE) == 1 || H5Tequal(dtype, H5T_IEEE_F32LE) == 1 || + H5Tequal(dtype, H5T_NATIVE_FLOAT) == 1) { + printf("data type: float\n"); + float *data = static_cast(malloc(sizeof(float) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else // H5T_ORDER_BE + status = H5Dread(dset, H5T_IEEE_F32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + /*Print the first 20 data values to check the correctness.*/ + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%f ", data[i]); + printf("\n"); + + // write out file data + if (0 > (idsid = H5Dcreate(fid, DATASET, H5T_IEEE_F32LE, sid, H5P_DEFAULT, cpid, H5P_DEFAULT))) + ERROR(H5Dcreate); + if (0 > H5Dwrite(idsid, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data)) ERROR(H5Dwrite); + if (0 > H5Dclose(idsid)) ERROR(H5Dclose); + + free(data); + } else // 64bit: double + { + printf("data type: double\n"); + double *data = static_cast(malloc(sizeof(double) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_IEEE_F64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + /*Print the first 10 data values to check the correctness.*/ + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%f ", data[i]); + printf("\n"); + free(data); + } + break; + case H5T_INTEGER: + if (0 > (dsign = H5Tget_sign(dtype))) { + printf("Error in calling H5Tget_sign(type_id)....\n"); + exit(0); + } + if (dsign == H5T_SGN_NONE) // unsigned + { + if (dsize == 1) { + printf("data type: unsigned char\n"); + unsigned char *data = static_cast(malloc(sizeof(unsigned char) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 2) { + printf("data type: unsigned short\n"); + unsigned short *data = static_cast(malloc(sizeof(unsigned short) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 4) { + printf("data type: unsigned int\n"); + unsigned int *data = static_cast(malloc(sizeof(unsigned int) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 8) { + printf("data type: unsigned long\n"); + uint64_t *data = static_cast(malloc(sizeof(uint64_t) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_U64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_U64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%" PRIu64 " ", data[i]); + printf("\n"); + free(data); + } + } else { + if (dsize == 1) { + printf("data type: char\n"); + char *data = static_cast(malloc(sizeof(char) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I8LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I8BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 2) { + printf("data type: short\n"); + short *data = static_cast(malloc(sizeof(short) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I16LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I16BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 4) { + printf("data type: int\n"); + int *data = static_cast(malloc(sizeof(int) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I32BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%d ", data[i]); + printf("\n"); + free(data); + } else if (dsize == 8) { + printf("data type: long\n"); + int64_t *data = static_cast(malloc(sizeof(int64_t) * nbEle)); + if (dorder == H5T_ORDER_LE) + status = H5Dread(dset, H5T_STD_I64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + else + status = H5Dread(dset, H5T_STD_I64BE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); + if (status < 0) READ_ERROR(); + printf("reconstructed data = "); + for (int i = 0; i < 20; i++) printf("%" PRId64 " ", data[i]); + printf("\n"); + free(data); + } + } + + break; + default: + printf("Error: H5Z-SZ3 supports only float, double or integers.\n"); + exit(1); + } + + // Close reading resources + status = H5Pclose(dcpl); + status = H5Dclose(dset); + status = H5Fclose(file); + + /*Close and release writing resources*/ + if (0 > H5Sclose(sid)) ERROR(H5Sclose); + if (0 > H5Pclose(cpid)) ERROR(H5Pclose); + if (0 > H5Fclose(fid)) ERROR(H5Fclose); + + return 0; +} diff --git a/tools/H5Z-SZ3/tools/sz3ToHDF5.cpp b/tools/H5Z-SZ3/tools/sz3ToHDF5.cpp new file mode 100644 index 00000000..33cc4e64 --- /dev/null +++ b/tools/H5Z-SZ3/tools/sz3ToHDF5.cpp @@ -0,0 +1,232 @@ +/** + * @file szToHDF5.c + * @author Sheng Di + * @date July, 2017 + * @brief This is an example of using compression interface (HDF5) + * (C) 2017 by Mathematics and Computer Science (MCS), Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + */ + + +#include +#include +#include +#include + +#include "hdf5.h" +// #include "H5Cpp.h" +#include "H5Z_SZ3.hpp" +#include "SZ3/utils/Config.hpp" +#include "SZ3/utils/FileUtil.hpp" + +#define LITTLE_ENDIAN_SYSTEM 0 +#define BIG_ENDIAN_SYSTEM 1 +#define LITTLE_ENDIAN_DATA 0 +#define BIG_ENDIAN_DATA 1 +int sysEndianType = LITTLE_ENDIAN_SYSTEM; +int dataEndianType = LITTLE_ENDIAN_DATA; + +#define DATASET "testdata_compressed" + +int MAX_CHUNK_SIZE = INT_MAX; + +// detect sys endian type +inline void detectSysEndianType() { + // get sys endian type + int x_temp = 1; + char *y_temp = reinterpret_cast(&x_temp); + + if (*y_temp == 1) + sysEndianType = LITTLE_ENDIAN_SYSTEM; + else //=0 + sysEndianType = BIG_ENDIAN_SYSTEM; +} + +template +void process_data(const SZ3::Config &conf, const char *oriFilePath, int dataEndianType, hid_t fid, hid_t sid, + hid_t cpid, const char *datasetName, hid_t h5TypeLE, hid_t h5TypeBE) { + T *data = new T[conf.num]; + SZ3::readfile(oriFilePath, conf.num, data); + + std::cout << "original data = "; + for (int i = 0; i < 20; ++i) std::cout << data[i] << " "; + std::cout << "....\n"; + + hid_t dataset; + hid_t h5Type = (dataEndianType == LITTLE_ENDIAN_DATA) ? h5TypeLE : h5TypeBE; + // The filter compresses host-order values only; H5Dwrite converts from the input's byte order. + hid_t storedType = H5Tget_native_type(h5Type, H5T_DIR_DEFAULT); + + dataset = H5Dcreate(fid, datasetName, storedType, sid, H5P_DEFAULT, cpid, H5P_DEFAULT); + H5Tclose(storedType); + if (dataset < 0) { + std::cerr << "Error in H5Dcreate\n"; + delete[] data; + exit(EXIT_FAILURE); + } + if (H5Dwrite(dataset, h5Type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) { + std::cerr << "Error in H5Dwrite\n"; + delete[] data; + exit(EXIT_FAILURE); + } + + delete[] data; + if (H5Dclose(dataset) < 0) { + std::cerr << "Error in H5Dclose\n"; + exit(EXIT_FAILURE); + } +} + +int main(int argc, char *argv[]) { + char oriFilePath[640], outputFilePath[1024]; + + if (argc < 3) { + printf("Test case: sz3ToHDF5 [dataType] [srcFilePath] [dimension sizes...]\n"); + printf("Example1 : sz3ToHDF5 -f testdata/x86/testfloat_8_8_128.dat 8 8 128\n"); + printf("Example 2: sz3ToHDF5 -i32 testdata/x86/testint32_8x8x8.dat 8 8 8\n"); + exit(0); + } + + std::map dataTypeMap = { + {"-f", SZ_FLOAT}, {"-d", SZ_DOUBLE}, {"-i8", SZ_INT8}, {"-u8", SZ_UINT8}, {"-i16", SZ_INT16}, + {"-u16", SZ_UINT16}, {"-i32", SZ_INT32}, {"-u32", SZ_UINT32}, {"-i64", SZ_INT64}, {"-u64", SZ_UINT64}}; + + int dataType = 0; + auto it = dataTypeMap.find(argv[1]); + if (it != dataTypeMap.end()) { + dataType = it->second; + } else { + std::cerr << "Error: unknown data type in sz3ToHDF5.c!\n"; + return 0; + } + + snprintf(oriFilePath, 640, "%s", argv[2]); + + std::vector dimensions; + for (int i = 3; i < argc && i < 8; ++i) { + dimensions.push_back(std::atoi(argv[i])); + } + std::reverse(dimensions.begin(), dimensions.end()); // slowest to fastest + + snprintf(outputFilePath, 1024, "%s.sz3.h5", oriFilePath); + + hid_t sid, cpid, fid; + /* create HDF5 file */ + if (0 > (fid = H5Fcreate(outputFilePath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT))) { + printf("Error in H5Pcreate"); + exit(0); + } + + // set up SZ configuration + SZ3::Config conf; + // use config.loadcfg("path_to_sz3_conf") to load configuration from a file if needed + // setting up data related attributes (data type, dims, etc.) is not necessary, as they will be updated in + // H5Z_sz3_set_local automatically + conf.setDims(dimensions.begin(), dimensions.end()); + // Set compression related attributes here + // conf.cmprAlgo = SZ3::ALGO_BIOMD; + + std::vector hdims(conf.dims.begin(), conf.dims.end()); + /*Create dataspace. Setting maximum size */ + if (0 > (sid = H5Screate_simple(conf.N, hdims.data(), NULL))) { + printf("Error in H5Screate_simple"); + exit(0); + } + /* setup dataset creation properties */ + if (0 > (cpid = H5Pcreate(H5P_DATASET_CREATE))) { + printf("Error in H5Pcreate"); + exit(0); + } + + set_SZ3_conf_to_H5(cpid, conf); + + if (0 > H5Zregister(H5PLget_plugin_info())) { + printf("Error in H5Zregister"); + exit(0); + } + if (H5Zfilter_avail(H5Z_FILTER_SZ3)) { + unsigned filter_config; + if (H5Zget_filter_info(H5Z_FILTER_SZ3, &filter_config) & H5Z_FILTER_CONFIG_ENCODE_ENABLED) + printf("sz filter is available for encoding and decoding.\n"); + } + + /* set the chunk size*/ + std::vector hchunk(hdims); + // hchunk[0] = 10; + if (0 > H5Pset_chunk(cpid, conf.N, hchunk.data())) { + printf("Error in H5Pcreate"); + exit(0); + } + + { // This is an example to get/set SZ configuration from HDF5 file + SZ3::Config conf1; + get_SZ3_conf_from_H5(cpid, conf1); + // conf1.absErrorBound = 1; + set_SZ3_conf_to_H5(cpid, conf1); + } + printf("....Writing SZ compressed data.............\n"); + + switch (dataType) { + case SZ_FLOAT: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_IEEE_F32LE, + H5T_IEEE_F32BE); + break; + case SZ_DOUBLE: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_IEEE_F64LE, + H5T_IEEE_F64BE); + break; + case SZ_INT8: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I8LE, + H5T_STD_I8BE); + break; + case SZ_UINT8: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U8LE, + H5T_STD_U8BE); + break; + case SZ_INT16: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I16LE, + H5T_STD_I16BE); + break; + case SZ_UINT16: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U16LE, + H5T_STD_U16BE); + break; + case SZ_INT32: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I32LE, + H5T_STD_I32BE); + break; + case SZ_UINT32: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U32LE, + H5T_STD_U32BE); + break; + case SZ_INT64: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_I64LE, + H5T_STD_I64BE); + break; + case SZ_UINT64: + process_data(conf, oriFilePath, dataEndianType, fid, sid, cpid, DATASET, H5T_STD_U64LE, + H5T_STD_U64BE); + break; + default: + std::cerr << "Error: Unknown data type\n"; + exit(EXIT_FAILURE); + } + + /*Close and release resources*/ + if (0 > H5Sclose(sid)) { + printf("Error in H5Sclose"); + exit(0); + } + if (0 > H5Pclose(cpid)) { + printf("Error in H5Pclose"); + exit(0); + } + if (0 > H5Fclose(fid)) { + printf("Error in H5Fclose"); + exit(0); + } + printf("Output hdf5 file: %s\n", outputFilePath); + if (H5Zunregister(H5Z_FILTER_SZ3) < 0) return -1; + H5close(); + return 0; +} diff --git a/tools/mdz/CMakeLists.txt b/tools/mdz/CMakeLists.txt new file mode 100644 index 00000000..fe0bfff4 --- /dev/null +++ b/tools/mdz/CMakeLists.txt @@ -0,0 +1,20 @@ +if (SZ3_INSTALL) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/MDZ) +endif () + +set(exes mdz + mdz_smoke_test) +foreach (EXE IN LISTS exes) + message(STATUS "Target = ${EXE}") + add_executable(${EXE} ${EXE}.cpp) + target_include_directories(${EXE} PUBLIC + "$" + "$" + ) + target_link_libraries(${EXE} SZ3) + # As in tools/sz3: the spelled-out path misses both the per-config subdirectory and the .exe. + if (SZ3_INSTALL) + install(PROGRAMS $ TYPE BIN) + endif () +endforeach () + diff --git a/tools/mdz/README.md b/tools/mdz/README.md new file mode 100644 index 00000000..76c1bbf3 --- /dev/null +++ b/tools/mdz/README.md @@ -0,0 +1,37 @@ +MMD-SZ(MDZ): A Modular Error-bounded Lossy Compressor Optimized for Molecular Dynamics +===== +(C) 2021 by Mathematics and Computer Science (MCS), Argonne National Laboratory. +See COPYRIGHT in top-level directory. + +* Major Authors: Kai Zhao, Sheng Di, Danny Perez +* Supervisor: Franck Cappello + +## Citations +* Kai Zhao, Sheng Di, Danny Perez, Zizhong Chen, and Franck Cappello. "[MDZ: An Efficient Error-bounded Lossy Compressor for Molecular Dynamics Simulations](https://ieeexplore.ieee.org/document/9835212)", Proceeding of the 38th IEEE International Conference on Data Engineering (ICDE 22), Kuala Lumpur, Malaysia, May 9 - + 12, 2022. + +## Installation + +Build SZ3 with the cmake option "-DBUILD_MDZ=ON" +`mdz` and `mdz_smoke_test` are installed in `[INSTALL_DIR]/bin` and the header files in `[INSTALL_DIR]/include`. + +`mdz` compresses and decompresses in memory and prints the compression ratio, the times and the error; it does not write a compressed file. + +## Testing Examples +mdz datafile -2 dim1 dim2 -r reb buffer_size compressor + +mdz datafile -3 dim1 dim2 dim3 -r reb buffer_size compressor + +#### options: +* datafile: FP32 binary format. Contains single axis (X or Y or Z) only. +* dim1: number of timesteps +* dim2: number of atoms +* dim3: number of atom dimensions (x,y,z,etc.) +* reb: relative error bound, for example, 1E-3; `-a ` instead of `-r ` takes an absolute error bound +* buffer_size (optional): default 10 +* compressor (optional): -1: ADP (chooses one of the others), 0: VQ, 1: VQT, 2: MT, 3: Lorenzo+Regression, 4: TS + +#### examples: +* mdz helium-mode-b-7852x1037/x.f32.dat -2 7852 1037 -r 1E-3 +* mdz helium-mode-b-7852x1037/x.f32.dat -2 7852 1037 -r 1E-3 10 +* mdz helium-mode-b-7852x1037/xyz.f32.dat -3 7852 1037 3 -r 1E-3 10 diff --git a/tools/mdz/include/mdz.hpp b/tools/mdz/include/mdz.hpp new file mode 100644 index 00000000..c9a535e8 --- /dev/null +++ b/tools/mdz/include/mdz.hpp @@ -0,0 +1,508 @@ +// +// Created by Kai Zhao on 7/1/21. +// + +#ifndef SZ3_MDZ_H +#define SZ3_MDZ_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/compressor/specialized/SZExaaltCompressor.hpp" +#include "SZ3/decomposition/BlockwiseDecomposition.hpp" +#include "SZ3/decomposition/TimeSeriesDecomposition.hpp" +#include "SZ3/utils/KmeansUtil.hpp" + +using namespace SZ3; +inline double total_compress_time = 0; +inline double total_decompress_time = 0; +inline const char *compressor_names[] = {"VQ", "VQT", "MT", "LR", "TS"}; + +template +std::shared_ptr> make_sz2(const Config &conf, Predictor predictor) { + return make_compressor_sz_generic( + make_decomposition_blockwise(conf, predictor, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), Lossless_zstd()); + // return new SZGeneralCompressor>, + // HuffmanEncoder, Lossless_zstd>( + // SZGeneralFrontend>(conf, predictor, + // LinearQuantizer(conf.absErrorBound, + // conf.quantbinCnt / 2)), + // HuffmanEncoder(), + // Lossless_zstd()); +} + +template +std::shared_ptr> make_sz_timebased2(const Config &conf, Predictor predictor, + T *data_ts0) { + return make_compressor_sz_generic( + make_decomposition_timeseries(conf, predictor, + LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), data_ts0), + HuffmanEncoder(), Lossless_zstd()); + // return std::make_shared>, + // HuffmanEncoder, Lossless_zstd>( + // TimeSeriesDecomposition>(conf, predictor, + // LinearQuantizer(conf.absErrorBound, + // conf.quantbinCnt / 2), data_ts0), + // HuffmanEncoder(), + // Lossless_zstd()); +} + +template +std::shared_ptr> make_sz_timebased(const Config &conf, T *data_ts0) { + std::vector>> predictors; + + int use_single_predictor = (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1; + if (conf.lorenzo) { + if (use_single_predictor) { + return make_sz_timebased2(conf, LorenzoPredictor(conf.absErrorBound), data_ts0); + } else { + predictors.push_back(std::make_shared>(conf.absErrorBound)); + } + } + if (conf.lorenzo2) { + if (use_single_predictor) { + return make_sz_timebased2(conf, LorenzoPredictor(conf.absErrorBound), data_ts0); + } else { + predictors.push_back(std::make_shared>(conf.absErrorBound)); + } + } + if (conf.regression) { + if (use_single_predictor) { + return make_sz_timebased2(conf, RegressionPredictor(conf.blockSize, conf.absErrorBound), + data_ts0); + } else { + predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); + } + } + + return make_sz_timebased2(conf, ComposedPredictor(predictors), data_ts0); +} + +template +std::shared_ptr> make_sz(const Config &conf) { + std::vector>> predictors; + + int use_single_predictor = (conf.lorenzo + conf.lorenzo2 + conf.regression) == 1; + if (conf.lorenzo) { + if (use_single_predictor) { + return make_sz2(conf, LorenzoPredictor(conf.absErrorBound)); + } else { + predictors.push_back(std::make_shared>(conf.absErrorBound)); + } + } + if (conf.lorenzo2) { + if (use_single_predictor) { + return make_sz2(conf, LorenzoPredictor(conf.absErrorBound)); + } else { + predictors.push_back(std::make_shared>(conf.absErrorBound)); + } + } + if (conf.regression) { + if (use_single_predictor) { + return make_sz2(conf, RegressionPredictor(conf.blockSize, conf.absErrorBound)); + } else { + predictors.push_back(std::make_shared>(conf.blockSize, conf.absErrorBound)); + } + } + + return make_sz2(conf, ComposedPredictor(predictors)); +} + +template +float *VQ(Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, int method, float level_start, + float level_offset, int level_num) { + if (level_num == 0) { + throw std::runtime_error("VQ/VQT not availble on current dataset, please use ADP or MT"); + } + + auto sz = make_compressor_exaalt(LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), + HuffmanEncoder(), Lossless_zstd(), method); + sz->set_level(level_start, level_offset, level_num); + + Timer timer(true); + compressed_size = 2 * conf.num * sizeof(T); + auto compressed = static_cast(malloc(compressed_size)); + // compress() returns what it actually wrote; compressed_size went in as the capacity, and the + // decompression below reads it as the stream length. + compressed_size = sz->compress(conf, data, compressed, compressed_size); + total_compress_time += timer.stop("Compression"); + if (!decom) { + free(compressed); + return nullptr; + } + // auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; + // std::cout << "Compression Ratio = " << ratio << std::endl; + // std::cout << "Compressed size = " << compressed_size << std::endl; + + timer.start(); + auto ts_dec_data = new T[conf.num]; + sz->decompress(conf, compressed, compressed_size, ts_dec_data); + total_decompress_time += timer.stop("Decompression"); + + free(compressed); + return ts_dec_data; +} + +template +float *MT(Config conf, size_t ts, T *data, size_t &compressed_size, bool decom, T *ts0) { + // printf("eb=%.8f\n", conf.eb); + auto sz = make_sz_timebased(conf, ts0); + + Timer timer(true); + compressed_size = 2 * conf.num * sizeof(T); + auto compressed = static_cast(malloc(compressed_size)); + // compress() returns what it actually wrote; compressed_size went in as the capacity, and the + // decompression below reads it as the stream length. + compressed_size = sz->compress(conf, data, compressed, compressed_size); + total_compress_time += timer.stop("Compression"); + if (!decom) { + free(compressed); + return nullptr; + } + + // auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; + // std::cout << "Compression Ratio = " << ratio << std::endl; + // std::cout << "Compressed size = " << compressed_size << std::endl; + + timer.start(); + auto ts_dec_data = new T[conf.num]; + sz->decompress(conf, compressed, compressed_size, ts_dec_data); + total_decompress_time += timer.stop("Decompression"); + + // delete sz; + free(compressed); + return ts_dec_data; +} + +template +float *SZ2(Config conf, size_t ts, T *data, size_t &compressed_size, bool decom) { + auto sz = make_sz(conf); + + Timer timer(true); + compressed_size = 2 * conf.num * sizeof(T); + auto compressed = static_cast(malloc(compressed_size)); + // compress() returns what it actually wrote; compressed_size went in as the capacity, and the + // decompression below reads it as the stream length. + compressed_size = sz->compress(conf, data, compressed, compressed_size); + + total_compress_time += timer.stop("Compression"); + if (!decom) { + free(compressed); + return nullptr; + } + + // auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; + // std::cout << "Compression Ratio = " << ratio << std::endl; + // std::cout << "Compressed size = " << compressed_size << std::endl; + + timer.start(); + auto ts_dec_data = new T[conf.num]; + sz->decompress(conf, compressed, compressed_size, ts_dec_data); + total_decompress_time += timer.stop("Decompression"); + + // delete sz; + free(compressed); + return ts_dec_data; +} + +template +void select(Config conf, int &method, size_t ts, T *data_all, float level_start, float level_offset, int level_num, + T *data_ts0, size_t batch_size) { + // && (ts_last_select == -1 || t - ts_last_select >= conf.timestep_batch * 10)) { + // std::cout << "****************** BEGIN Selection ****************" << std::endl; + // ts_last_select = ts; + std::vector compressed_size(10, std::numeric_limits::max()); + std::vector data1; + size_t t = ts; + if (ts == 0) { + if (conf.dims[0] == 1) { // if the data shape is (1, XXX), then no need for testing + method = (level_num > 0 ? 0 : 3); + return; + } else { // if first batch, only use the second half of the batch for testing + t = conf.dims[0] / 2; + conf.dims[0] /= 2; + } + } + if (batch_size > 10) { + conf.dims[0] = 10; + } + conf.num = conf.dims[0] * conf.dims[1]; + + // std::cout << conf.dims[0] << " " << conf.dims[1] << " " << t << std::endl; + + if (level_num > 0) { + data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num); + VQ(conf, t, data1.data(), compressed_size[0], false, 0, level_start, level_offset, level_num); + + data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num); + VQ(conf, t, data1.data(), compressed_size[1], false, 1, level_start, level_offset, level_num); + } else { + data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num); + SZ2(conf, t, data1.data(), compressed_size[3], false); + } + + data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num); + MT(conf, t, data1.data(), compressed_size[2], false, data_ts0); + + // data1 = std::vector(&data_all[t * conf.dims[1]], &data_all[t * conf.dims[1]] + conf.num); + // MT(conf, t, data1.data(), compressed_size[4], false, (T *) nullptr); + + method = std::distance(compressed_size.begin(), std::min_element(compressed_size.begin(), compressed_size.end())); + // printf("Select %s as Compressor, timestep=%lu, method=%d\n", + // compressor_names[method], + // ts, method); + // std::cout << "****************** END Selection ****************" << std::endl; +} + +template +std::unique_ptr readfile(const char *file, size_t start, size_t num) { + std::ifstream fin(file, std::ios::binary); + if (!fin) { + std::cout << " Error, Couldn't find the file" + << "\n"; + return nullptr; + } + fin.seekg(0, std::ios::end); + // const size_t total_num_elements = fin.tellg() / sizeof(Type); + // assert(start + num <= total_num_elements); + fin.seekg(start * sizeof(Type), std::ios::beg); + auto data = std::make_unique(num); + fin.read(reinterpret_cast(&data[0]), num * sizeof(Type)); + fin.close(); + return data; +} + +template +uchar *LAMMPS_compress(Config conf, T *data, int method, size_t &compressed_size, float level_start, float level_offset, + int level_num, T *ts0) { + if ((method == 0 || method == 1) && level_num == 0) { + throw std::runtime_error("VQ/VQT not available on current dataset, please use ADP or MT"); + } + compressed_size = 2 * conf.num * sizeof(T); + auto compressed_data = new uchar[compressed_size]; + std::shared_ptr> sz; + if (method == 0 || method == 1) { + sz = make_compressor_exaalt(LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2), + HuffmanEncoder(), Lossless_zstd(), method); + sz->set_level(level_start, level_offset, level_num); + } else if (method == 2 || method == 4) { + sz = make_sz_timebased(conf, ts0); + // sz->compress(conf, data, compressed_size); + } else { + sz = make_sz(conf); + // sz->compress(conf, data, compressed_size); + } + // compressed_size is the caller's out-parameter for the stream length, not the capacity it went in as. + compressed_size = sz->compress(conf, data, compressed_data, compressed_size); + // auto ratio = conf.num * sizeof(T) * 1.0 / compressed_size; + // std::cout << "Compression Ratio = " << ratio << std::endl; + // std::cout << "Compressed size = " << compressed_size << std::endl; + return compressed_data; +} + +template +int LAMMPS_select_compressor(Config conf, T *data, bool firsttime, float level_start, float level_offset, int level_num, + T *data_ts0) { + // std::cout << "****************** BEGIN Selection ****************" << std::endl; + + std::vector compressed_size(10, std::numeric_limits::max()); + + if (firsttime) { + conf.dims[0] /= 2; + conf.num = conf.dims[0] * conf.dims[1]; + data += conf.num; + } + if (conf.dims[0] > 10) { + conf.dims[0] = 10; + conf.num = conf.dims[0] * conf.dims[1]; + } + + std::vector data1; + uchar *cmpr; + if (level_num > 0) { + data1 = std::vector(data, data + conf.num); + cmpr = LAMMPS_compress(conf, data1.data(), 0, compressed_size[0], level_start, level_offset, level_num, + data_ts0); + delete[] cmpr; + + data1 = std::vector(data, data + conf.num); + cmpr = LAMMPS_compress(conf, data1.data(), 1, compressed_size[1], level_start, level_offset, level_num, + data_ts0); + delete[] cmpr; + } else { + data1 = std::vector(data, data + conf.num); + cmpr = LAMMPS_compress(conf, data1.data(), 3, compressed_size[3], level_start, level_offset, level_num, + data_ts0); + delete[] cmpr; + } + + data1 = std::vector(data, data + conf.num); + cmpr = LAMMPS_compress(conf, data1.data(), 2, compressed_size[2], level_start, level_offset, level_num, + data_ts0); + delete[] cmpr; + + // data1 = std::vector(&data[t * conf.dims[1]], &data[t * conf.dims[1]] + conf.num); + // MT(conf, t, data1.data(), compressed_size[4], false, (T *) nullptr); + + int method = + std::distance(compressed_size.begin(), std::min_element(compressed_size.begin(), compressed_size.end())); + printf("Select %s as Compressor, method=%d\n", compressor_names[method], method); + // std::cout << "****************** END Selection ****************" << std::endl; + return method; +} + +template +inline typename std::enable_if::type MDZ_Compress(Config conf, T *input_data, T *dec_data, + size_t batch_size, int method = -1) { + // if (N != 2) { + // throw std::invalid_argument("dimension should be 2"); + // } + if (batch_size == 0) { + batch_size = conf.dims[0]; + } + int method_batch = 0; + if (method == -1) { + method_batch = 50; + } + // std::cout << "****************** Options ********************" << std::endl; + // std::cout << "dimension = " << N + // << ", error bound = " << conf.absErrorBound + // << ", method = " << method + // << ", method_update_batch = " << method_batch + // << ", batch_size = " << batch_size + // << ", quan_state_num = " << conf.quantbinCnt + // // << ", encoder = " << conf.encoder_op + // // << ", lossless = " << conf.lossless_op + // << std::endl; + + // auto data_all = readfile(input_path.data(), 0, conf.num); + // auto data_all = input_data; + auto data_ts0 = std::vector(input_data, input_data + conf.dims[1]); + + float level_start, level_offset; + int level_num = 0; + if (method != 2 && method != 3 && method != 4) { + size_t sample_num = 0.1 * conf.dims[1]; + sample_num = std::min(sample_num, static_cast(20000)); + sample_num = std::max(sample_num, std::min(static_cast(5000), conf.dims[1])); + get_cluster(input_data, conf.dims[1], level_start, level_offset, level_num, sample_num); + if (level_num > conf.dims[1] * 0.25) { + level_num = 0; + } + // if (level_num != 0) { + // printf("start = %.3f , level_offset = %.3f, nlevel=%d\n", level_start, level_offset, level_num); + // } + } + + auto dims = conf.dims; + // auto total_num = conf.num; + size_t total_compressed_size = 0; + int current_method = method; + bool lossless_first_frame = false; + for (size_t ts = 0; ts < dims[0]; ts += batch_size) { + conf.dims[0] = (ts + batch_size > dims[0] ? dims[0] - ts : batch_size); + conf.num = conf.dims[0] * conf.dims[1]; + + T *data = &input_data[ts * conf.dims[1]]; + + T max = *std::max_element(data, data + conf.num); + T min = *std::min_element(data, data + conf.num); + if (conf.errorBoundMode == EB_ABS) { + conf.relErrorBound = conf.absErrorBound / (max - min); + } else if (conf.errorBoundMode == EB_REL) { + conf.absErrorBound = conf.relErrorBound * (max - min); + } + + // std::cout << "****************** Compression From " << ts << " to " << ts + conf.dims[0] - 1 + // << " ******************" << std::endl; + // std::cout< 0 && ts / batch_size % method_batch == 0) { + select(conf, current_method, ts, input_data, level_start, level_offset, level_num, data_ts0.data(), + batch_size); + } + // printf("Compressor = %s\n", compressor_names[current_method]); + + std::cout << "From " << ts << " to " << ts + conf.dims[0] - 1 + << " , Compressor = " << compressor_names[current_method] << std::endl; + + T *ts_dec_data; + size_t compressed_size; + if (current_method == 0) { + ts_dec_data = + VQ(conf, ts, data, compressed_size, true, current_method, level_start, level_offset, level_num); + } else if (current_method == 1) { + ts_dec_data = + VQ(conf, ts, data, compressed_size, true, current_method, level_start, level_offset, level_num); + } else if (current_method == 2) { + lossless_first_frame = true; + ts_dec_data = MT(conf, ts, data, compressed_size, true, data_ts0.data()); + } else if (current_method == 4) { + ts_dec_data = MT(conf, ts, data, compressed_size, true, static_cast(nullptr)); + } else { + ts_dec_data = SZ2(conf, ts, data, compressed_size, true); + } + total_compressed_size += compressed_size; + memcpy(&dec_data[ts * conf.dims[1]], ts_dec_data, conf.num * sizeof(T)); + } + if (lossless_first_frame) { + auto zstd = SZ3::Lossless_zstd(); + size_t inSize = conf.dims[1] * sizeof(T); + uchar *buffer = new uchar[inSize * 2]; + auto cmpSize = zstd.compress(reinterpret_cast(data_ts0.data()), inSize, buffer, inSize * 2); + delete[] buffer; + // printf("outsize %lu\n", cmpSize); + total_compressed_size += cmpSize; + } + + return total_compressed_size; +} + +template +inline typename std::enable_if::type MDZ_Compress(Config conf, T *input_data, T *dec_data, + size_t batch_size, int method = -1) { + size_t total_compressed_size = 0; + auto dims = conf.dims; + std::vector input(conf.num), output(conf.num); + for (size_t frame = 0; frame < conf.dims[0]; frame++) { + for (size_t atom = 0; atom < conf.dims[1]; atom++) { + for (size_t xyz = 0; xyz < conf.dims[2]; xyz++) { + input[xyz * dims[0] * dims[1] + frame * dims[1] + atom] = + input_data[frame * dims[1] * dims[2] + atom * dims[2] + xyz]; + } + } + } + for (size_t i = 0; i < conf.dims[2]; i++) { + Config conf_2D(conf); + conf_2D.dims = {conf.dims[0], conf.dims[1]}; + conf_2D.num = conf.dims[0] * conf.dims[1]; + total_compressed_size += + MDZ_Compress(conf_2D, input.data() + i * conf.dims[0] * conf.dims[1], + output.data() + i * conf.dims[0] * conf.dims[1], batch_size, method); + } + for (size_t frame = 0; frame < conf.dims[0]; frame++) { + for (size_t atom = 0; atom < conf.dims[1]; atom++) { + for (size_t xyz = 0; xyz < conf.dims[2]; xyz++) { + dec_data[frame * dims[1] * dims[2] + atom * dims[2] + xyz] = + output[xyz * dims[0] * dims[1] + frame * dims[1] + atom]; + } + } + } + return total_compressed_size; +} + +#endif // SZ3_MDZ_H diff --git a/tools/mdz/mdz.cpp b/tools/mdz/mdz.cpp new file mode 100644 index 00000000..ae8af22d --- /dev/null +++ b/tools/mdz/mdz.cpp @@ -0,0 +1,89 @@ +#include +#include + +inline void usage() { + printf("Usage: \n"); + printf("For 2D input: mdz file_path -2 n_frames n_atoms -r reb\n"); + printf("For 3D input: mdz file_path -3 n_frames n_atoms n_dims(x,y,z) -r reb\n"); + exit(0); +} + +int main(int argc, char **argv) { + if (argc < 2) { + usage(); + } + + if (argc < 3) { + usage(); + } + int dim = atoi(argv[2] + 1); + if (dim < 2 || dim > 3) { + usage(); + } + int argp = 3; + std::vector dims(dim); + for (int i = 0; i < dim; i++) { + dims[i] = atoi(argv[argp++]); + } + + SZ3::Config conf({dims[0], dims[1]}); + if (dim == 3) { + conf = SZ3::Config({dims[0], dims[1], dims[2]}); + } + + std::string input_path = argv[1]; + + if (argp >= argc) { + usage(); + } + char *eb_op = argv[argp++] + 1; + if (*eb_op == 'a') { + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = atof(argv[argp++]); + } else { + conf.errorBoundMode = SZ3::EB_REL; + conf.relErrorBound = atof(argv[argp++]); + } + size_t batch_size = 0; + if (argp < argc) { + batch_size = atoi(argv[argp++]); + } + int method = -1; + if (argp < argc) { + method = atoi(argv[argp++]); + } + + conf.blockSize = 128; + conf.quantbinCnt = 1024; + // conf.enable_regression = false; + // conf.quant_state_num = 4096; + if (argp < argc) { + conf.quantbinCnt = atoi(argv[argp++]); + } + auto input_data = readfile(input_path.data(), 0, conf.num); + std::vector input2(input_data.get(), input_data.get() + conf.num); + std::vector dec_data(conf.num); + + size_t compressed_size = 0; + if (conf.N == 2) { + compressed_size = MDZ_Compress(conf, input_data.get(), dec_data.data(), batch_size, method); + } else if (conf.N == 3) { + compressed_size = MDZ_Compress(conf, input_data.get(), dec_data.data(), batch_size, method); + } + float ratio = conf.num * 1.0 * sizeof(float) / compressed_size; + + double max_diff, psnr, nrmse; + printf("\nBatch=%lu\nCompression ratio=%.3f\nCompression time=%.3f\nDecompression time=%.3f\n", + (batch_size == 0 ? dims[0] : batch_size), ratio, total_compress_time, total_decompress_time); + + SZ3::verify(input2.data(), dec_data.data(), conf.num, psnr, nrmse, max_diff); + // std::cout << "****************** Final ****************" << std::endl; + // printf("method=md, file=%s, block=%lu, compression_ratio=%.3f, reb=%.1e, eb=%.6f, psnr=%.3f, nsmse=%e, + // compress_time=%.3f, decompress_time=%.3f, timestep_op=%d\n", + // input_path.data(), batch_size, + // ratio, + // conf.relErrorBound, + // max_diff, psnr, nrmse, + // total_compress_time, total_decompress_time, + // method); +} diff --git a/tools/mdz/mdz_smoke_test.cpp b/tools/mdz/mdz_smoke_test.cpp new file mode 100644 index 00000000..1124e071 --- /dev/null +++ b/tools/mdz/mdz_smoke_test.cpp @@ -0,0 +1,31 @@ +// +// Created by Kai Zhao on 11/10/22. +// + +#include + +int main(int argc, char **argv) { + std::vector dims({100, 200}); + SZ3::Config conf({dims[0], dims[1]}); + + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 1e-6; + // conf.blockSize = 128; + // conf.stride = 128; + conf.quantbinCnt = 1024; + + std::vector input_data(conf.num); + std::vector dec_data(conf.num); + + for (size_t i = 0; i < dims[0]; ++i) { + for (size_t j = 0; j < dims[1]; ++j) { + double x = static_cast(i) - static_cast(dims[0]) / 2.0; + double y = static_cast(j) - static_cast(dims[1]) / 2.0; + input_data[i * dims[1] + j] = static_cast(.0001 * y * sin(y) + .0005 * cos(pow(x, 2) + x)); + } + } + + size_t compressed_size = MDZ_Compress(conf, input_data.data(), dec_data.data(), 10); + printf("compression ratio = %.5f", dims[0] * dims[1] * sizeof(float) * 1.0 / compressed_size); + return 0; +} diff --git a/tools/paraview/CMakeLists.txt b/tools/paraview/CMakeLists.txt new file mode 100644 index 00000000..00dcfba6 --- /dev/null +++ b/tools/paraview/CMakeLists.txt @@ -0,0 +1,27 @@ +find_package(ParaView REQUIRED) + +# Enable the SZ3Reader plugin by default +set(PARAVIEW_PLUGIN_ENABLE_SZ3Reader ON CACHE BOOL "Enable SZ3Reader plugin") + +# Scan the plugin. +paraview_plugin_scan( + PLUGIN_FILES "${CMAKE_CURRENT_SOURCE_DIR}/SZ3Reader/paraview.plugin" + PROVIDES_PLUGINS plugins + REQUIRES_MODULES modules) + +# Ensure we have required modules. +foreach (module IN LISTS modules) + if (NOT TARGET "${module}") + message(FATAL_ERROR + "The required ${module} module is not available in the " + "ParaView build.") + endif () +endforeach () + +# Build the plugin. +paraview_plugin_build( + RUNTIME_DESTINATION "${CMAKE_INSTALL_BINDIR}" + LIBRARY_DESTINATION "${CMAKE_INSTALL_LIBDIR}" + LIBRARY_SUBDIRECTORY "${PARAVIEW_PLUGIN_SUBDIR}" + TARGET paraview_plugins + PLUGINS ${plugins}) \ No newline at end of file diff --git a/tools/paraview/README.md b/tools/paraview/README.md new file mode 100644 index 00000000..5229519e --- /dev/null +++ b/tools/paraview/README.md @@ -0,0 +1,61 @@ +# SZ3Reader Plugin +ParaView/VTK reader for direct visualization of data files compressed using the SZ3 compressor. + +Version: 0.1 + +General information +-------------------- +For more information on SZ3, see . + +For more information on ParaView, see . + +The SZ3Reader plugin is built on the SZ3 C++ API. It reads SZ3-compressed files (e.g., `.sz3`). + +The current version supports regular data with up to three dimensions in single- or double-precision floating-point format. + + +Installation +------------- + +Before the plugin is included in the official ParaView release, you will need to manually build the plugin from sources and load it in ParaView. + +#### Prerequisites + +- Download the ParaView sources from . If you have ParaView already installed, please make sure you select the same version from the drop-down list. +- Follow the [ParaView building process](https://www.paraview.org/paraview-docs/latest/cxx/md__builds_gitlab-kitware-sciviz-ci_Documentation_dev_build.html). + +#### Building the Plugin + +1. Configure the SZ3 project with CMake and enable the ParaView plugin: + + ```bash + cmake -DBUILD_PARAVIEW_PLUGIN=ON [other options] .. + ``` + +2. If CMake does not find ParaView automatically, set `ParaView_DIR` as `[-DParaView_DIR=/path/to/paraview/lib/cmake/paraview-]`. The option `PARAVIEW_PLUGIN_ENABLE_SZ3Reader` will be set to `ON` by default. + +3. Build the project (optional: use parallel jobs with `-j`): + + ```bash + cmake --build . -j [num_threads] + ``` + +#### Loading the Plugin in ParaView + +1. After the build completes, open ParaView. +2. Navigate to `Tools -> Manage Plugins -> Load New`. +3. Browse to and select the plugin file: + - **Linux/macOS**: `{SZ3_BUILD_DIR}/lib/paraview-{VERSION}/plugins/SZ3Reader/SZ3Reader.so` + - **Windows**: `{SZ3_BUILD_DIR}/lib/paraview-{VERSION}/plugins/SZ3Reader/SZ3Reader.dll` +4. Check `Auto Load` if you want the plugin to be loaded automatically every time you run ParaView. + + +Contact Info +------------- + +This plugin can certainly be improved and any suggestions or contributions are welcome. + +Please report bugs, problems, and suggestions to: + +- Guoxi Liu (email: [liu.12722@osu.edu](mailto:liu.12722@osu.edu)) +- Siheng Zhang (email: [zhang.16458@buckeyemail.osu.edu](mailto:zhang.16458@buckeyemail.osu.edu)) diff --git a/tools/paraview/SZ3Reader/CMakeLists.txt b/tools/paraview/SZ3Reader/CMakeLists.txt new file mode 100644 index 00000000..98ac01d6 --- /dev/null +++ b/tools/paraview/SZ3Reader/CMakeLists.txt @@ -0,0 +1,13 @@ + +paraview_add_plugin(SZ3Reader + REQUIRED_ON_CLIENT + REQUIRED_ON_SERVER + VERSION "0.1" + MODULES SZ3Reader::vtkSZ3Reader + MODULE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/Reader/vtk.module" +) + +# Skip testing build for now +# if (BUILD_TESTING) +# add_subdirectory(Testing) +# endif () diff --git a/tools/paraview/SZ3Reader/Reader/CMakeLists.txt b/tools/paraview/SZ3Reader/Reader/CMakeLists.txt new file mode 100644 index 00000000..d8bfb49b --- /dev/null +++ b/tools/paraview/SZ3Reader/Reader/CMakeLists.txt @@ -0,0 +1,12 @@ +set(classes + vtkSZ3Reader) + +vtk_module_add_module(SZ3Reader::vtkSZ3Reader + CLASSES ${classes}) +vtk_module_link(SZ3Reader::vtkSZ3Reader + PUBLIC + SZ3) + +paraview_add_server_manager_xmls( + XMLS + SZ3Reader.xml) diff --git a/tools/paraview/SZ3Reader/Reader/SZ3Reader.xml b/tools/paraview/SZ3Reader/Reader/SZ3Reader.xml new file mode 100644 index 00000000..b2c43de6 --- /dev/null +++ b/tools/paraview/SZ3Reader/Reader/SZ3Reader.xml @@ -0,0 +1,48 @@ + + + + + + + + + + The name of the file to read. + + + + + + + Specify the dimensions of the domain (X, Y, Z). + + + + + + + Check to use double precision (float64). If unchecked, single precision (float32) will be used. + + + + + + + + + + diff --git a/tools/paraview/SZ3Reader/Reader/vtk.module b/tools/paraview/SZ3Reader/Reader/vtk.module new file mode 100644 index 00000000..f0c28f89 --- /dev/null +++ b/tools/paraview/SZ3Reader/Reader/vtk.module @@ -0,0 +1,11 @@ +NAME + SZ3Reader::vtkSZ3Reader +LIBRARY_NAME + vtkSZ3Reader +DESCRIPTION + "VTK module for reading SZ3 compressed files" +DEPENDS + VTK::CommonExecutionModel + ParaView::VTKExtensionsCore + VTK::CommonCore + VTK::CommonDataModel diff --git a/tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.cxx b/tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.cxx new file mode 100644 index 00000000..da724c26 --- /dev/null +++ b/tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.cxx @@ -0,0 +1,150 @@ +#include "vtkSZ3Reader.h" + +#include "vtkCellData.h" +#include "vtkDataArray.h" +#include "vtkAOSDataArrayTemplate.h" +#include "vtkImageAlgorithm.h" +#include "vtkImageData.h" +#include "vtkInformation.h" +#include "vtkInformationVector.h" +#include "vtkObjectFactory.h" +#include "vtkPVInformationKeys.h" +#include "vtkPointData.h" +#include "vtkStreamingDemandDrivenPipeline.h" + +#include +#include +#include + +vtkStandardNewMacro(vtkSZ3Reader) + +vtkSZ3Reader::vtkSZ3Reader() +{ + this->FileName = nullptr; + this->DomainDimensions[0] = 0; + this->DomainDimensions[1] = 0; + this->DomainDimensions[2] = 0; + this->UseDoublePrecision = 0; + this->SetNumberOfInputPorts(0); + this->SetNumberOfOutputPorts(1); +} + +vtkSZ3Reader::~vtkSZ3Reader() +{ + this->SetFileName(nullptr); +} + +void vtkSZ3Reader::SetDomainDimensions(int x, int y, int z) +{ + this->DomainDimensions[0] = x; + this->DomainDimensions[1] = y; + this->DomainDimensions[2] = z; + this->Modified(); +} + +void vtkSZ3Reader::GetDomainDimensions(int& x, int& y, int& z) +{ + x = this->DomainDimensions[0]; + y = this->DomainDimensions[1]; + z = this->DomainDimensions[2]; +} + +int vtkSZ3Reader::RequestData( + vtkInformation* /*request*/, vtkInformationVector** /*inputVector*/, vtkInformationVector* outputVector) +{ + + if (!this->FileName) { + vtkErrorMacro("A FileName must be specified."); + return 0; + } + + if (this->DomainDimensions[0] <= 0 || this->DomainDimensions[1] <= 0 || this->DomainDimensions[2] <= 0) { + vtkErrorMacro("Dimensions must be greater than zero."); + return 0; + } + + std::ifstream file(this->FileName, std::ios::binary); + if(!file) { + vtkErrorMacro("Could not open file: " << this->FileName); + return 0; + } + + file.seekg(0, std::ios::end); + size_t compressedSize = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector compressedBuffer(compressedSize); + file.read(compressedBuffer.data(), compressedSize); + file.close(); + + auto conf = SZ3::Config(DomainDimensions[0], DomainDimensions[1], DomainDimensions[2]); + + vtkImageData* output = vtkImageData::GetData(outputVector); + output->SetDimensions(this->DomainDimensions); + + if (this->UseDoublePrecision) { + this->Decompress(output, conf, compressedBuffer); + } + else { + this->Decompress(output, conf, compressedBuffer); + } + + return 1; +} + +template +void vtkSZ3Reader::Decompress( + vtkImageData* output, SZ3::Config& conf, std::vector& compressedBuffer) +{ + using VtkArrayT = vtkAOSDataArrayTemplate; + vtkNew dataArray; + + dataArray->SetNumberOfComponents(1); + dataArray->SetNumberOfTuples(conf.num); + dataArray->SetName("scalar"); + + // Decompress directly into the VTK array's buffer + T* decompressedPtr = static_cast(dataArray->GetVoidPointer(0)); + + try { + SZ_decompress( + conf, reinterpret_cast(compressedBuffer.data()), compressedBuffer.size(), decompressedPtr); + } + catch (std::exception& e) { + vtkErrorMacro("Decompression failed: " << e.what()); + return; + } + + output->GetPointData()->AddArray(dataArray); + output->GetPointData()->SetScalars(dataArray); +} + +int vtkSZ3Reader::RequestInformation( + vtkInformation* /*request*/, + vtkInformationVector** /*inputVector*/, + vtkInformationVector* outputVector) +{ + vtkInformation* outInfo = outputVector->GetInformationObject(0); + + // Set the whole extent + int extent[6] = {0, 0, 0, 0, 0, 0}; + extent[1] = this->DomainDimensions[0] > 0 ? this->DomainDimensions[0] - 1 : 0; + extent[3] = this->DomainDimensions[1] > 0 ? this->DomainDimensions[1] - 1 : 0; + extent[5] = this->DomainDimensions[2] > 0 ? this->DomainDimensions[2] - 1 : 0; + outInfo->Set(vtkStreamingDemandDrivenPipeline::WHOLE_EXTENT(), extent, 6); + + // Set default origin and spacing + double origin[3] = {0.0, 0.0, 0.0}; + double spacing[3] = {1.0, 1.0, 1.0}; + outInfo->Set(vtkDataObject::ORIGIN(), origin, 3); + outInfo->Set(vtkDataObject::SPACING(), spacing, 3); + + return 1; +} + +void vtkSZ3Reader::PrintSelf(ostream& os, vtkIndent indent) +{ + this->Superclass::PrintSelf(os, indent); + os << indent << "FileName: " << (this->FileName ? this->FileName : "(none)") << "\n"; + os << indent << "DomainDimensions: " << this->DomainDimensions[0] << ", " << this->DomainDimensions[1] << ", " << this->DomainDimensions[2] << "\n"; + os << indent << "UseDoublePrecision: " << this->UseDoublePrecision << "\n"; +} diff --git a/tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.h b/tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.h new file mode 100644 index 00000000..0dd1377b --- /dev/null +++ b/tools/paraview/SZ3Reader/Reader/vtkSZ3Reader.h @@ -0,0 +1,45 @@ +#ifndef vtkSZ3Reader_h +#define vtkSZ3Reader_h + +#include "vtkSZ3ReaderModule.h" // for export macro +#include "vtkDataObjectAlgorithm.h" +#include "vtkImageAlgorithm.h" + +#include +#include + +class VTKSZ3READER_EXPORT vtkSZ3Reader : public vtkImageAlgorithm +{ +public: + static vtkSZ3Reader* New(); + vtkTypeMacro(vtkSZ3Reader, vtkImageAlgorithm); + void PrintSelf(ostream& os, vtkIndent indent) override; + + vtkSetStringMacro(FileName); + vtkGetStringMacro(FileName); + + void SetDomainDimensions(int x, int y, int z); + void GetDomainDimensions(int& x, int& y, int& z); + + void SetDoublePrecision(int flag) { this->UseDoublePrecision = flag; this->Modified(); } + int GetDoublePrecision() const { return this->UseDoublePrecision; } + +protected: + vtkSZ3Reader(); + ~vtkSZ3Reader(); + int RequestInformation(vtkInformation*, vtkInformationVector**, vtkInformationVector*) override; + int RequestData(vtkInformation*, vtkInformationVector**, vtkInformationVector*) override; + + char* FileName; + int DomainDimensions[3]; + int UseDoublePrecision; + +private: + vtkSZ3Reader(const vtkSZ3Reader&); + void operator=(const vtkSZ3Reader&); + + template + void Decompress(vtkImageData* output, SZ3::Config& conf, std::vector& compressedBuffer); +}; + +#endif diff --git a/tools/paraview/SZ3Reader/paraview.plugin b/tools/paraview/SZ3Reader/paraview.plugin new file mode 100644 index 00000000..df520b10 --- /dev/null +++ b/tools/paraview/SZ3Reader/paraview.plugin @@ -0,0 +1,10 @@ +NAME + SZ3Reader +DESCRIPTION + Reader for direct visualization of data using the SZ3 compressor. +REQUIRES_MODULES + ParaView::VTKExtensionsCore + VTK::CommonCore + VTK::CommonDataModel + VTK::CommonExecutionModel + VTK::vtksys diff --git a/tools/pysz/.gitignore b/tools/pysz/.gitignore new file mode 100644 index 00000000..2e6d0a7b --- /dev/null +++ b/tools/pysz/.gitignore @@ -0,0 +1,32 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +*.so +*.dylib +*.dll + +# Cython generated files +*.c +*.cpp +*.html + +# Build directories +build/ +dist/ +*.egg-info/ +.eggs/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# macOS +.DS_Store + +# Test outputs +*.dat.sz +*.dat.sz.out diff --git a/tools/pysz/LICENSE b/tools/pysz/LICENSE new file mode 100644 index 00000000..e49b4777 --- /dev/null +++ b/tools/pysz/LICENSE @@ -0,0 +1,36 @@ +Copyright Š 2016 , UChicago Argonne, LLC +All Rights Reserved +[SZ, Version 3.0] +Sheng Di +Kai Zhao +Xin Liang +Dingwen Tao +Franck Cappello +Argonne National Laboratory + +SZ licence is under the BSD license. Details are shown below. + +OPEN SOURCE LICENSE (license number: SF-16-105) + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Software changes, modifications, or derivative works, should be noted with comments and the author and organization's name. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the names of UChicago Argonne, LLC or the Department of Energy nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +4. The software and the end-user documentation included with the redistribution, if any, must include the following acknowledgment: + + "This product includes software produced by UChicago Argonne, LLC under Contract No. DE-AC02-06CH11357 with the Department of Energy." + +****************************************************************************************************** +DISCLAIMER + +THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. + +NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. + +*************************************************************************************************** + +Contact: Sheng Di (sdi1@anl.gov), Franck Cappello(cappello@mcs.anl.gov) diff --git a/tools/pysz/MANIFEST.in b/tools/pysz/MANIFEST.in new file mode 100644 index 00000000..2a12b24a --- /dev/null +++ b/tools/pysz/MANIFEST.in @@ -0,0 +1,28 @@ +# Include README and LICENSE files +include README.md +include LICENSE + +# Include all Cython source files +include src/pysz/*.pyx +include src/pysz/*.pxd +include src/pysz/*.py + +# Include tests +include tests/*.py + +# Include build configuration +include pyproject.toml +include setup.py + +# Exclude build artifacts and cache +global-exclude __pycache__ +global-exclude *.py[cod] +global-exclude *.so +global-exclude *.dylib +global-exclude .DS_Store + +# Exclude Cython-generated files (will be regenerated during build) +global-exclude *.c +global-exclude *.cpp +global-exclude *.html + diff --git a/tools/pysz/Makefile b/tools/pysz/Makefile new file mode 100644 index 00000000..ebbe1f35 --- /dev/null +++ b/tools/pysz/Makefile @@ -0,0 +1,34 @@ +.PHONY: install build clean test help + +help: + @echo "pysz - Python bindings for SZ3" + @echo "" + @echo "Available targets:" + @echo " install - Install package in editable mode" + @echo " build - Build the package" + @echo " clean - Remove all build artifacts" + @echo " test - Run tests" + @echo " help - Show this help message" + +install: + pip install -e . + +build: + python -m build + +clean: + @echo "Cleaning build artifacts..." + rm -rf build/ dist/ .eggs/ *.egg-info + rm -rf src/*.egg-info + find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true + find . -type f -name "*.pyc" -delete + find . -type f -name "*.pyo" -delete + find . -type f -name "*.so" -delete + find . -type f -name "*.dylib" -delete + find . -type f -name "*.cpp" -path "*/pysz/*" -delete + find . -type f -name "*.c" -path "*/pysz/*" -delete + find . -type f -name "*.html" -delete + @echo "✓ Cleaned!" + +test: + python tests/test_pysz.py diff --git a/tools/pysz/README.md b/tools/pysz/README.md new file mode 100644 index 00000000..865d74b8 --- /dev/null +++ b/tools/pysz/README.md @@ -0,0 +1,263 @@ +# pysz + +Python bindings for SZ3 - Error-bounded lossy compression for scientific data. + +## Overview + +pysz provides a clean Python interface to SZ3, a fast error-bounded lossy compressor for scientific data. Built with Cython 3.0+ for high performance. + +## Installation + +### From PyPI (recommended) + +**Pre-built binary wheels** are available for most platforms (Linux, macOS, Windows) and Python versions (3.9-3.14): + +```bash +pip install pysz +``` + +This is the easiest method - no build tools required! The binary wheels include everything you need. + +### Building from source + +If pre-built wheels aren't available for your platform, or if you want to build from source: + +**You need to first install the following tools:** +- **CMake â‰Ĩ 3.19** +- **C++ compiler** (C++17-compatible: g++, clang++, MSVC) +- **Git** +- **Python development headers** (python3-dev or python3-devel) + +**Then build pysz from pip:** +```bash +pip install pysz +``` +**Or build pysz from source:** +```bash +git clone https://github.com/szcompressor/SZ3.git +cd SZ3/tools/pysz +pip install -e . +``` + +**Or build pysz against an SZ3 you have installed** (with its Zstd in the same prefix): +```bash +PYSZ_SZ3_PREFIX= pip install . +``` + +**What happens during source installation:** +1. SZ3 is automatically downloaded from GitHub +2. SZ3 is built with CMake (zstd is bundled) +3. Python bindings are compiled against the built SZ3 +4. Everything is packaged together + + +## Quick Start + +```python +import numpy as np +from pysz import sz, szConfig, szErrorBoundMode, szAlgorithm + +# Create test data +data = np.random.rand(8, 8, 128).astype(np.float32) + +# Create config +config = szConfig() +config.errorBoundMode = szErrorBoundMode.ABS +config.absErrorBound = 1e-3 + +# Compress +compressed, ratio = sz.compress(data, config) +print(f"Compression ratio: {ratio:.2f}x") + +# Decompress +decompressed, config = sz.decompress(compressed, np.float32, data.shape) + +# Verify +max_err, psnr, nrmse = sz.verify(data, decompressed) +print(f"Max error: {max_err:.2e}, PSNR: {psnr:.2f} dB, NRMSE: {nrmse:.2e}") +``` + +## API Reference + +### szConfig Class + +Configuration object for SZ3 compression. It mirrors the C++ `Config` class. +During compression, the dimensions are automatically inferred from the numpy array shape, so you don't need setup the config dimensions. + +### sz.compress() + +```python +sz.compress(data, config) -> (compressed, ratio) +``` + +Compress a NumPy array. Dimensions are automatically inferred from data shape. + +**Parameters:** +- `data` (ndarray): NumPy array (float32, float64, int32, or int64) +- `config` (szConfig): Config object + +**Returns:** +- `compressed` (ndarray): Compressed data as uint8 array +- `ratio` (float): Compression ratio (original_size / compressed_size) + +### sz.decompress() + +```python +sz.decompress(compressed, dtype, shape) -> (data, config) +``` + +Decompress data back to NumPy array. + +**Parameters:** +- `compressed` (ndarray): Compressed uint8 array from `compress()` +- `dtype` (type): NumPy dtype (np.float32, np.float64, np.int32, or np.int64) +- `shape` (tuple): Shape of the original data. It must hold as many values as the compressed data, or `ValueError` is raised. + +**Returns:** +- `data` (ndarray): Decompressed data with the specified shape +- `config` (szConfig): Config object used for decompression + +### sz.verify() + +```python +sz.verify(src_data, dec_data) -> (max_diff, psnr, nrmse) +``` + +Compare decompressed data with original data and calculate quality metrics. + +**Parameters:** +- `src_data` (ndarray): Original data before compression +- `dec_data` (ndarray): Decompressed data to verify + +**Returns:** +- `max_diff` (float): Maximum absolute difference +- `psnr` (float): Peak Signal-to-Noise Ratio in dB +- `nrmse` (float): Normalized Root Mean Square Error + +**Example:** +```python +>>> max_err, psnr, nrmse = sz.verify(data, decompressed) +>>> print(f"Max error: {max_err:.2e}, PSNR: {psnr:.2f} dB") +``` + +## Usage Examples + +### Basic Usage + +```python +import numpy as np +from pysz import sz, szConfig, szErrorBoundMode, szAlgorithm + +# Create data +data = np.random.randn(100, 200, 300).astype(np.float32) + +# Create config +config = szConfig() +config.errorBoundMode = szErrorBoundMode.ABS +config.absErrorBound = 1e-3 + +# Compress +compressed, ratio = sz.compress(data, config) +print(f"Compressed {data.nbytes} → {compressed.size} bytes ({ratio:.2f}x)") + +# Decompress - returns (data, config) +decompressed, dec_config = sz.decompress(compressed, np.float32, data.shape) + +# Verify quality +max_err, psnr, nrmse = sz.verify(data, decompressed) +print(f"Max error: {max_err:.2e}, PSNR: {psnr:.2f} dB, NRMSE: {nrmse:.2e}") +``` + +### Loading from Config File (Optional) + +```python +import numpy as np +from pysz import sz, szConfig + +data = np.fromfile('testdata.dat', dtype=np.float32).reshape(8, 8, 128) + +# Load config from file +config = szConfig() +config.loadcfg('sz3.config') +compressed, ratio = sz.compress(data, config) +decompressed, _ = sz.decompress(compressed, np.float32, data.shape) + +``` + +### Relative Error Bound + +```python +from pysz import szConfig, szErrorBoundMode + +config = szConfig() +config.errorBoundMode = szErrorBoundMode.REL +config.relErrorBound = 1e-4 # 0.01% relative error +compressed, ratio = sz.compress(data, config) +``` + + +### Different Algorithms + +```python +from pysz import szConfig, szErrorBoundMode, szAlgorithm + +# Default: INTERP_LORENZO (best quality) +config = szConfig() +config.errorBoundMode = szErrorBoundMode.ABS +config.absErrorBound = 1e-3 + +# Or try other algorithms +config.cmprAlgo = szAlgorithm.INTERP # Interpolation only +config.cmprAlgo = szAlgorithm.LORENZO_REG # Lorenzo/regression +config.cmprAlgo = szAlgorithm.LOSSLESS # Lossless only +``` + +### Double Precision + +```python +data_double = np.random.randn(50, 50, 50).astype(np.float64) +config = szConfig() +config.errorBoundMode = szErrorBoundMode.ABS +config.absErrorBound = 1e-6 +compressed, ratio = sz.compress(data_double, config) +decompressed, _ = sz.decompress(compressed, np.float64, data_double.shape) +``` + +### Save/Load Compressed Data + +```python +# Compress and save +compressed, ratio = sz.compress(data, config) +compressed.tofile('data.sz') + +# Later: load and decompress +compressed = np.fromfile('data.sz', dtype=np.uint8) +decompressed, _ = sz.decompress(compressed, np.float32, (8, 8, 128)) +``` + +## Compatibility + +pysz 1.1 is built on SZ3 3.4.0. It decompresses data compressed by pysz 1.0 (SZ3 3.3.2), except data compressed with `szAlgorithm.NOPRED` and integer (int32, int64) data; decompress those with pysz 1.0. See the SZ3 [CHANGELOG](https://github.com/szcompressor/SZ3/blob/master/CHANGELOG.md). + +## Troubleshooting + +### Import Error + +``` +ImportError: cannot import name 'sz' from 'pysz' +``` + +**Solution:** +```bash +cd tools/pysz +pip install -e . +``` + +## Links + +- **Repository:** https://github.com/szcompressor/SZ3 +- **Issues:** https://github.com/szcompressor/SZ3/issues + +## License + +See `../../copyright-and-BSD-license.txt` diff --git a/tools/pysz/deprecated/cpysz.py b/tools/pysz/deprecated/cpysz.py new file mode 100644 index 00000000..b7fa27a1 --- /dev/null +++ b/tools/pysz/deprecated/cpysz.py @@ -0,0 +1,133 @@ +import sys +import ctypes +import numpy as np + +""" +Python API for SZ3 +""" + + +class SZ: + def __init__(self, szpath=None): + """ + init SZ + :param szpath: the path to SZ dynamic library + """ + + if szpath is None: + szpath = { + "darwin": "libSZ3c.dylib", + "win32": "SZ3c.dll", + }.get(sys.platform, "libSZ3c.so") + + self.sz = ctypes.cdll.LoadLibrary(szpath) + + self.sz.SZ_compress_args.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.POINTER(ctypes.c_size_t), + ctypes.c_int, ctypes.c_double, ctypes.c_double, ctypes.c_double, + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, + ctypes.c_size_t) + self.sz.SZ_compress_args.restype = ctypes.POINTER(ctypes.c_ubyte) + + self.sz.SZ_decompress.argtypes = (ctypes.c_int, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t, + ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, ctypes.c_size_t, + ctypes.c_size_t) + + self.sz.free_buf.argtype = (ctypes.c_void_p) + + def __sz_datatype(self, dtype, data=None): + if dtype == np.float32: + return 0, data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) if data is not None else None + elif dtype == np.float64: + return 1, data.ctypes.data_as(ctypes.POINTER(ctypes.c_double)) if data is not None else None + else: + print('SZ currently supports float32 and float64\n') + exit(0) + + def verify(self, src_data, dec_data): + """ + Compare the decompressed data with original data + :param src_data: original data, numpy array + :param dec_data: decompressed data, numpy array + :return: max_diff, psnr, nrmse + """ + data_range = np.max(src_data) - np.min(src_data) + diff = src_data - dec_data + max_diff = np.max(abs(diff)) + print("abs err={:.8G}".format(max_diff)) + mse = np.mean(diff ** 2) + nrmse = np.sqrt(mse) / data_range + psnr = 20 * np.log10(data_range) - 10 * np.log10(mse) + return max_diff, psnr, nrmse + + def decompress(self, data_cmpr, original_shape, original_dtype): + """ + Decompress data with SZ + :param data_cmpr: compressed data, numpy array format, dtype should be np.uint8 + :param original_shape: the shape of original data + :param original_dtype: the dtype of original data + :return: decompressed data,numpy array format + """ + + r5, r4, r3, r2, r1 = [0] * (5 - len(original_shape)) + list(original_shape) + ori_type, ori_null = self.__sz_datatype(original_dtype) + self.sz.SZ_decompress.restype = ctypes.POINTER( + ctypes.c_float if original_dtype == np.float32 else ctypes.c_double) + data_dec_c = self.sz.SZ_decompress(ori_type, + data_cmpr.ctypes.data_as(ctypes.POINTER(ctypes.c_ubyte)), + data_cmpr.size, + r5, r4, r3, r2, r1) + + # Calculate total number of elements + total_elements = np.prod(original_shape) + + # Use np.frombuffer for fast memory copy + # Cast the ctypes pointer to void pointer, then use addressof to get memory address + buffer_ptr = ctypes.cast(data_dec_c, ctypes.c_void_p) + buffer_size = total_elements * np.dtype(original_dtype).itemsize + + # Create numpy array directly from memory buffer + data_dec = np.frombuffer((ctypes.c_ubyte * buffer_size).from_address(buffer_ptr.value), + dtype=original_dtype, + count=total_elements + ).reshape(original_shape) + + # Make a copy since we're going to free the original memory + data_dec = data_dec.copy() + + self.sz.free_buf(data_dec_c) + return data_dec + + def compress(self, data, eb_mode, eb_abs, eb_rel, eb_pwr): + """ + Compress data with SZ + :param data: original data, numpy array format, dtype is FP32 or FP64 + :param eb_mode:# error bound mode, integer (0: ABS, 1:REL, 2:ABS_AND_REL, 3:ABS_OR_REL, 4:PSNR, 5:NORM, 10:PW_REL) + :param eb_abs: optional, abs error bound, double + :param eb_rel: optional, rel error bound, double + :param eb_pwr: optional, pwr error bound, double + :return: compressed data, numpy array format, dtype is np.uint8 + compression ratio + """ + assert len(data.shape) <= 5, "SZ only supports 1D to 5D input data" + cmpr_size = ctypes.c_size_t() + r5, r4, r3, r2, r1 = [0] * (5 - len(data.shape)) + list(data.shape) + datatype, datap = self.__sz_datatype(data.dtype, data) + data_cmpr_c = self.sz.SZ_compress_args(datatype, datap, + ctypes.byref(cmpr_size), + eb_mode, eb_abs, eb_rel, eb_pwr, + r5, r4, r3, r2, r1) + + cmpr_ratio = data.size * data.itemsize / cmpr_size.value + + # Use np.frombuffer for fast memory copy + buffer_ptr = ctypes.cast(data_cmpr_c, ctypes.c_void_p) + buffer_size = cmpr_size.value + + # Create numpy array directly from memory buffer + data_cmpr = np.frombuffer((ctypes.c_ubyte * buffer_size).from_address(buffer_ptr.value), + dtype=np.uint8, + count=buffer_size + ).copy() # Make a copy since we're going to free the original memory + + self.sz.free_buf(data_cmpr_c) + return data_cmpr, cmpr_ratio diff --git a/tools/pysz/deprecated/test.py b/tools/pysz/deprecated/test.py new file mode 100644 index 00000000..99c2d160 --- /dev/null +++ b/tools/pysz/deprecated/test.py @@ -0,0 +1,25 @@ +import numpy as np +from cpysz import SZ +import sys + +# prepare your data in numpy array format +data = np.random.uniform(low=0, high=1, size=(100, 100)) + +# init SZ (both SZ2 and SZ3 are supported) +# Please change the path to the SZ dynamic library file in your system +lib_extention = { + "darwin": "libSZ3c.dylib", + "windows": "SZ3c.dll", +}.get(sys.platform, "libSZ3c.so") + +sz = SZ("../../install/lib/{}".format(lib_extention)) + +# compress, both input and output data are numpy array +data_cmpr, cmpr_ratio = sz.compress(data, 0, 1e-3, 0, 0) +print("compression ratio = {:5G}".format(cmpr_ratio)) + +# decompress, both input and output data are numpy array +data_dec = sz.decompress(data_cmpr, data.shape, data.dtype) + +# verify +sz.verify(data, data_dec) diff --git a/tools/pysz/pyproject.toml b/tools/pysz/pyproject.toml new file mode 100644 index 00000000..ae82e8e3 --- /dev/null +++ b/tools/pysz/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = [ + "setuptools>=61.0", + "cython>=3.0.10", + "numpy>=1.20.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "pysz" +version = "1.1.0" +authors = [ + { name = "SZ3 Team" }, +] +description = "Python interface for SZ3: An error-bounded lossy compressor for scientific data" +readme = "README.md" +requires-python = ">=3.9" +license = {text = "BSD-3-Clause"} +keywords = ["compression", "scientific data", "lossy compression", "SZ3", "error-bounded"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: C++", + "Programming Language :: Cython", + "Topic :: Scientific/Engineering", + "Topic :: System :: Archiving :: Compression", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", +] +dependencies = [ + "numpy>=1.20.0", +] + +[project.urls] +"Homepage" = "https://github.com/szcompressor/SZ3" +"Documentation" = "https://github.com/szcompressor/SZ3/tree/master/tools/pysz" +"Source" = "https://github.com/szcompressor/SZ3" +"Bug Tracker" = "https://github.com/szcompressor/SZ3/issues" + diff --git a/tools/pysz/setup.py b/tools/pysz/setup.py new file mode 100644 index 00000000..70aea57b --- /dev/null +++ b/tools/pysz/setup.py @@ -0,0 +1,166 @@ +""" +Setup script for pysz - Python bindings for SZ3 +Automatically downloads and builds SZ3 with bundled zstd, or with PYSZ_SZ3_PREFIX set, uses an installed SZ3 and the +Zstd installed beside it. +""" + +import os +import sys +import shutil +import subprocess +from pathlib import Path +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext as _build_ext +from Cython.Build import cythonize +import numpy as np + + + +# A released tag: a published wheel has to be buildable from a published source. +SZ3_VERSION = "3.4.0" + +# Both layouts, because the bundled Zstd's name and location changed after v3.3.2. +ZSTD_HEADER_DIRS = (("tools", "zstd", "lib"), ("build", "_deps", "zstdfetched-src", "lib")) +ZSTD_LIBRARY_DIRS = (("build", "tools", "zstd"), + ("build", "tools", "zstd", "Release"), + ("build", "tools", "zstd", "Debug")) +ZSTD_LIBRARY_NAMES = ("sz3_zstd", "zstd") + + +def find_zstd_header_dir(sz3_dir): + for parts in ZSTD_HEADER_DIRS: + candidate = sz3_dir.joinpath(*parts) + if (candidate / "zstd.h").is_file(): + return candidate + raise RuntimeError(f"no bundled zstd.h under {sz3_dir}; is SZ3_VERSION a tag that bundles Zstd?") + + +def find_zstd_library(sz3_dir): + """The link name of the bundled Zstd this SZ3 build produced, and the directory holding it.""" + for name in ZSTD_LIBRARY_NAMES: + for parts in ZSTD_LIBRARY_DIRS: + directory = sz3_dir.joinpath(*parts) + for pattern in (f"lib{name}.*", f"{name}.lib"): + if any(directory.glob(pattern)): + return name, directory + raise RuntimeError(f"no bundled Zstd library under {sz3_dir / 'build' / 'tools' / 'zstd'}") + + +class BuildSZ3Extension(_build_ext): + + def run(self): + sz3_prefix = os.environ.get("PYSZ_SZ3_PREFIX") + if sz3_prefix: + # As a distribution builds it: SZ3's headers, and the Zstd it was built with, from one prefix. + prefix = Path(sz3_prefix) + zstd_name = "sz3_zstd" if any((prefix / "lib").glob("*sz3_zstd*")) else "zstd" + for ext in self.extensions: + ext.include_dirs.insert(0, str(prefix / "include")) + ext.library_dirs.append(str(prefix / "lib")) + ext.libraries.append(zstd_name) + super().run() + return + + sz3_dir = self.download_and_build_sz3() + zstd_name, zstd_dir = find_zstd_library(sz3_dir) + print(f"Linking bundled Zstd: {zstd_name} from {zstd_dir}") + + for ext in self.extensions: + ext.include_dirs.insert(0, str(sz3_dir / "include")) + ext.include_dirs.insert(0, str(sz3_dir / "build" / "include")) + ext.include_dirs.append(str(find_zstd_header_dir(sz3_dir))) + ext.libraries.append(zstd_name) + ext.library_dirs.append(str(zstd_dir)) + + super().run() + + # A shared one has to ride along next to the extension; a static one is already in it. + package_dir = Path(self.build_lib) / "pysz" + if package_dir.exists(): + for pattern in (f"lib{zstd_name}.dylib", f"lib{zstd_name}.so", f"{zstd_name}.dll"): + for shared in sorted(zstd_dir.glob(pattern)): + shutil.copy2(shared, package_dir / shared.name) + print(f"Copied {shared.name} to package") + return + + def download_and_build_sz3(self): + build_temp = Path(self.build_temp).absolute() + build_temp.mkdir(parents=True, exist_ok=True) + sz3_dir = build_temp / "SZ3" + + if (sz3_dir / "build" / "include" / "SZ3" / "version.hpp").exists(): + print(f"SZ3 already built at: {sz3_dir}") + return sz3_dir + + if not sz3_dir.exists(): + print(f"Cloning SZ3 v{SZ3_VERSION}...") + subprocess.run([ + "git", "clone", "--depth", "1", + "--branch", f"v{SZ3_VERSION}", + "--single-branch", + "https://github.com/szcompressor/SZ3.git", + str(sz3_dir) + ], check=True) + + build_dir = sz3_dir / "build" + build_dir.mkdir(exist_ok=True) + + cmake_args = ["cmake"] + cmake_args.extend([ + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_TESTING=OFF", + "-DBUILD_SZ3_BINARY=OFF", + "-DSZ3_USE_BUNDLED_ZSTD=ON", + ".." + ]) + subprocess.run(cmake_args, cwd=build_dir, check=True) + subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True) + print(f"Built SZ3 v{SZ3_VERSION}") + return sz3_dir + + + + +def create_extensions(): + include_dirs = [np.get_include()] + library_dirs = [] + libraries = [] # the bundled Zstd is appended once the build knows what it is called + extra_compile_args = [] + extra_link_args = [] + + if sys.platform == 'win32': + extra_compile_args.extend(['/std:c++17', '/O2']) + elif sys.platform == 'darwin': + extra_compile_args.extend(['-std=c++17', '-O3', '-stdlib=libc++']) + extra_link_args.extend(['-stdlib=libc++', '-Wl,-rpath,@loader_path']) + elif sys.platform == 'linux': + extra_compile_args.extend(['-std=c++17', '-O3']) + extra_link_args.extend(['-Wl,-rpath,$ORIGIN']) + + extensions = [ + Extension( + "pysz.sz", + sources=["src/pysz/sz.pyx"], + include_dirs=include_dirs, + libraries=libraries, + library_dirs=library_dirs, + language='c++', + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + ), + ] + + return cythonize(extensions, compiler_directives={'language_level': '3', 'embedsignature': True}) + + +if __name__ == "__main__": + setup( + name="pysz", + version="1.1.0", + packages=["pysz"], + package_dir={"": "src"}, + ext_modules=create_extensions(), + cmdclass={'build_ext': BuildSZ3Extension}, + test_suite="tests", + tests_require=["pytest"], + ) diff --git a/tools/pysz/src/pysz/__init__.py b/tools/pysz/src/pysz/__init__.py new file mode 100644 index 00000000..6fdb732c --- /dev/null +++ b/tools/pysz/src/pysz/__init__.py @@ -0,0 +1,3 @@ +from pysz.sz import sz, szConfig, szErrorBoundMode, szAlgorithm + +__all__ = ["sz", "szConfig", "szErrorBoundMode", "szAlgorithm"] diff --git a/tools/pysz/src/pysz/sz.pxd b/tools/pysz/src/pysz/sz.pxd new file mode 100644 index 00000000..638abd50 --- /dev/null +++ b/tools/pysz/src/pysz/sz.pxd @@ -0,0 +1,66 @@ +"""Cython declarations for SZ3 compression/decompression API.""" + +from libcpp cimport bool +from libc.stdint cimport uint8_t, uint32_t, uint64_t, int32_t, int64_t +from libcpp.vector cimport vector +from libcpp.string cimport string +from libc.stddef cimport size_t + +cdef extern from "SZ3/utils/Config.hpp" namespace "SZ3": + cdef cppclass Config: + # Constructor + Config() except + + + # Methods + size_t setDims[Iter](Iter begin, Iter end) + void loadcfg(const string &cfgpath) except + + void load_ini(const string &ini_content) except + + string save_ini() const + size_t save(unsigned char *&c) const + void load(const unsigned char *&c, size_t &remaining_length) except + + void print() + size_t size_est() const + + # Basic configuration + uint32_t sz3MagicNumber + uint32_t sz3DataVer + char N + vector[size_t] dims + size_t num + + # Compression settings + uint8_t cmprAlgo + uint8_t errorBoundMode + double absErrorBound + double relErrorBound + double psnrErrorBound + double l2normErrorBound + bool openmp + + +cdef extern from "SZ3/utils/MemoryUtil.hpp" namespace "SZ3": + void read[T](T &var, const unsigned char *&pos) + + +cdef class szConfig: + cdef Config conf + + +cdef extern from "SZ3/api/sz.hpp" namespace "SZ3": + size_t SZ_compress_size_bound[T](const Config &conf) except + + + +cdef extern from "SZ3/api/sz.hpp": + # Compression with pre-allocated buffer + size_t SZ_compress[T](const Config &conf, + const T *data, + char *cmpData, + size_t cmpCap) except + + + # Decompression with pre-allocated buffer + # Note: Config is non-const because it gets loaded from compressed data + void SZ_decompress[T](Config &conf, + const char *cmpData, + size_t cmpSize, + T *&decData) except + + diff --git a/tools/pysz/src/pysz/sz.pyx b/tools/pysz/src/pysz/sz.pyx new file mode 100644 index 00000000..1db47024 --- /dev/null +++ b/tools/pysz/src/pysz/sz.pyx @@ -0,0 +1,433 @@ +# distutils: language = c++ +"""Python interface for SZ3 compression library.""" + +from pysz cimport sz as c_sz +cimport cython +from cython.operator cimport dereference +import numpy as np +cimport numpy as cnp +from libc.stdint cimport int8_t, int32_t, int64_t, uint64_t +from libc.stddef cimport size_t +from libc.string cimport memcpy +from typing import Tuple +from libcpp.vector cimport vector +from libcpp.string cimport string + +# Initialize NumPy C API +cnp.import_array() + + +class szErrorBoundMode: + """Error bound modes""" + ABS = 0 + REL = 1 + PSNR = 2 + L2NORM = 3 + ABS_AND_REL = 4 + ABS_OR_REL = 5 + + +class szAlgorithm: + """Compression algorithms""" + LORENZO_REG = 0 + INTERP_LORENZO = 1 + INTERP = 2 + NOPRED = 3 + LOSSLESS = 4 + + +cdef class szConfig: + """ + Configuration class for SZ3 compression. + + Enum classes: szErrorBoundMode, szAlgorithm (module level) + + Parameters: Dimension sizes as individual ints, tuple, or list + szConfig(data.shape), szConfig(100, 200, 300), etc. + """ + + def __init__(self, *args): + """Initialize config with optional dimensions.""" + self.conf = Config() + if args: + self.setDims(*args) + + def setDims(self, *args): + """Set dimensions. Accepts tuple/list or individual integers.""" + cdef vector[size_t] dims + + # Handle both setDims(100, 200, 300) and setDims((100, 200, 300)) + if len(args) == 1 and hasattr(args[0], '__iter__'): + dims_iter = args[0] + else: + dims_iter = args + + if not dims_iter: + raise ValueError("At least one dimension required") + + for arg in dims_iter: + if not isinstance(arg, int) or arg <= 0: + raise ValueError(f"Dimension must be positive integer, got {arg}") + dims.push_back(arg) + + self.conf.setDims(dims.begin(), dims.end()) + + def loadcfg(self, cfgpath: str): + """Load configuration from INI file.""" + cdef string cfgpathStr = cfgpath.encode('utf-8') + try: + self.conf.loadcfg(cfgpathStr) + except RuntimeError as e: + raise RuntimeError(f"Failed to load '{cfgpath}': {e}") + + # Read-only properties + @property + def dims(self): + """Get dimensions (read-only).""" + return tuple(self.conf.dims) + + @property + def num_elements(self): + """Get total number of elements (read-only).""" + return self.conf.num + + @property + def ndim(self): + """Get number of dimensions (read-only).""" + return self.conf.N + + # Error bounds (double) + @property + def absErrorBound(self): + return self.conf.absErrorBound + + @absErrorBound.setter + def absErrorBound(self, double value): + self.conf.absErrorBound = value + + @property + def relErrorBound(self): + return self.conf.relErrorBound + + @relErrorBound.setter + def relErrorBound(self, double value): + self.conf.relErrorBound = value + + @property + def psnrErrorBound(self): + return self.conf.psnrErrorBound + + @psnrErrorBound.setter + def psnrErrorBound(self, double value): + self.conf.psnrErrorBound = value + + @property + def l2normErrorBound(self): + return self.conf.l2normErrorBound + + @l2normErrorBound.setter + def l2normErrorBound(self, double value): + self.conf.l2normErrorBound = value + + # Enum properties (accept enum or int) + @property + def errorBoundMode(self): + """Get/set error bound mode. Use szErrorBoundMode enum.""" + return self.conf.errorBoundMode + + @errorBoundMode.setter + def errorBoundMode(self, value): + if isinstance(value, int): + self.conf.errorBoundMode = value + elif hasattr(value, 'value'): # Is an enum + self.conf.errorBoundMode = value.value + else: + raise TypeError(f"Expected int or Enum, got {type(value).__name__}") + + @property + def cmprAlgo(self): + """Get/set compression algorithm. Use szAlgorithm enum.""" + return self.conf.cmprAlgo + + @cmprAlgo.setter + def cmprAlgo(self, value): + if isinstance(value, int): + self.conf.cmprAlgo = value + elif hasattr(value, 'value'): # Is an enum + self.conf.cmprAlgo = value.value + else: + raise TypeError(f"Expected int or Enum, got {type(value).__name__}") + + @property + def openmp(self): + return self.conf.openmp + + @openmp.setter + def openmp(self, bint value): + self.conf.openmp = value + + def __repr__(self): + return f"szConfig(dims={self.dims}, num_elements={self.num_elements})" + + +cdef class sz: + """SZ3 compression/decompression with zero-copy NumPy API.""" + + # Supported dtypes + _SUPPORTED_DTYPES = frozenset([ + np.dtype(np.float32), + np.dtype(np.float64), + np.dtype(np.int32), + np.dtype(np.int64), + ]) + + @staticmethod + def compress(cnp.ndarray data, config) -> Tuple[cnp.ndarray, float]: + """ + Compress NumPy array using SZ3. + + Parameters + ---------- + data : numpy.ndarray + Input data (float32, float64, int32, or int64) + config : szConfig + Configuration object + + Returns + ------- + compressed : numpy.ndarray + Compressed data as uint8 array + ratio : float + Compression ratio (original_size / compressed_size) + """ + # Validate dtype + if data.dtype not in sz._SUPPORTED_DTYPES: + raise TypeError( + f"Unsupported dtype: {data.dtype}. " + f"Supported: float32, float64, int32, int64" + ) + + # Handle config parameter + # We need to cast the python object to our cdef class to access C++ internals + cdef szConfig conf + if isinstance(config, szConfig): + conf = config + # Always set dimensions based on input data + shape_tuple = tuple(data.shape[i] for i in range(data.ndim)) + conf.setDims(*shape_tuple) + else: + raise TypeError(f"config must be szConfig, got {type(config)}") + + # Ensure C-contiguous for zero-copy + if not data.flags['C_CONTIGUOUS']: + data = np.ascontiguousarray(data) + + # Get data pointer (zero-copy input!) + cdef void* data_ptr = cnp.PyArray_DATA(data) + cdef size_t original_size = data.nbytes + + cdef size_t buffer_size = 0 + if data.dtype == np.float32: + buffer_size = c_sz.SZ_compress_size_bound[float](conf.conf) + elif data.dtype == np.float64: + buffer_size = c_sz.SZ_compress_size_bound[double](conf.conf) + elif data.dtype == np.int32: + buffer_size = c_sz.SZ_compress_size_bound[int32_t](conf.conf) + elif data.dtype == np.int64: + buffer_size = c_sz.SZ_compress_size_bound[int64_t](conf.conf) + cdef cnp.ndarray[cnp.uint8_t, ndim=1] compressed = np.empty(buffer_size, dtype=np.uint8) + cdef char* compressed_ptr = cnp.PyArray_DATA(compressed) + cdef size_t compressed_size = 0 + + # Compress into pre-allocated buffer + if data.dtype == np.float32: + compressed_size = c_sz.SZ_compress[float]( + conf.conf, + data_ptr, + compressed_ptr, + buffer_size + ) + elif data.dtype == np.float64: + compressed_size = c_sz.SZ_compress[double]( + conf.conf, + data_ptr, + compressed_ptr, + buffer_size + ) + elif data.dtype == np.int32: + compressed_size = c_sz.SZ_compress[int32_t]( + conf.conf, + data_ptr, + compressed_ptr, + buffer_size + ) + elif data.dtype == np.int64: + compressed_size = c_sz.SZ_compress[int64_t]( + conf.conf, + data_ptr, + compressed_ptr, + buffer_size + ) + + if compressed_size == 0: + raise RuntimeError("Compression failed") + + # Resize to actual compressed size (no copy, just view) + compressed = compressed[:compressed_size] + + ratio = original_size / float(compressed_size) + return compressed, ratio + + + @staticmethod + def decompress(cnp.ndarray compressed, dtype, shape) -> Tuple[cnp.ndarray, szConfig]: + """ + Decompress SZ3-compressed data. + + Parameters + ---------- + compressed : numpy.ndarray + Compressed data (uint8 array) + dtype : numpy.dtype or type + Data type of original data + shape : tuple + Shape of the original data + + Returns + ------- + decompressed : numpy.ndarray + Decompressed data with the specified shape + config : szConfig + Configuration object used during decompression (contains actual compression params) + """ + # Validate compressed data + if compressed.dtype != np.uint8: + raise TypeError(f"Compressed data must be uint8, got {compressed.dtype}") + + # Validate and normalize dtype + dtype = np.dtype(dtype) + if dtype not in sz._SUPPORTED_DTYPES: + raise TypeError( + f"Unsupported dtype: {dtype}. " + f"Supported: float32, float64, int32, int64" + ) + + # Create a new config object for decompression + cdef szConfig conf = szConfig() + conf.setDims(*shape) + + # Ensure compressed data is contiguous + if not compressed.flags['C_CONTIGUOUS']: + compressed = np.ascontiguousarray(compressed) + + cdef char* compressed_ptr = cnp.PyArray_DATA(compressed) + cdef size_t compressed_size = compressed.size + cdef size_t num_elements = conf.num_elements + + # SZ_decompress writes as many values as the stream holds, so check them against shape. The 16-byte header + # ends with the payload size, and the Config follows the payload. + if compressed_size < 16: + raise ValueError("Compressed data is smaller than the SZ3 header") + cdef const unsigned char* pos = compressed_ptr + 8 + cdef uint64_t payload_size = 0 + c_sz.read[uint64_t](payload_size, pos) + if payload_size > compressed_size - 16: + raise ValueError("Compressed data is truncated") + pos += payload_size + cdef size_t remaining = compressed_size - 16 - payload_size + cdef c_sz.Config stored + stored.load(pos, remaining) + if stored.num != num_elements: + raise ValueError(f"shape {tuple(shape)} holds {num_elements} values, the compressed data {stored.num}") + + # Pre-allocate NumPy array for decompressed data + cdef cnp.ndarray result = np.empty(num_elements, dtype=dtype) + + # Get pointer to NumPy array data + # Pass this to SZ_decompress - it will decompress directly into our buffer! + cdef float* float_ptr + cdef double* double_ptr + cdef int32_t* int32_ptr + cdef int64_t* int64_ptr + + if dtype == np.float32: + float_ptr = cnp.PyArray_DATA(result) + c_sz.SZ_decompress[float]( + conf.conf, + compressed_ptr, + compressed_size, + float_ptr + ) + elif dtype == np.float64: + double_ptr = cnp.PyArray_DATA(result) + c_sz.SZ_decompress[double]( + conf.conf, + compressed_ptr, + compressed_size, + double_ptr + ) + elif dtype == np.int32: + int32_ptr = cnp.PyArray_DATA(result) + c_sz.SZ_decompress[int32_t]( + conf.conf, + compressed_ptr, + compressed_size, + int32_ptr + ) + elif dtype == np.int64: + int64_ptr = cnp.PyArray_DATA(result) + c_sz.SZ_decompress[int64_t]( + conf.conf, + compressed_ptr, + compressed_size, + int64_ptr + ) + + # Data is now in our NumPy array! (zero-copy decompression) + # Reshape to original dimensions + return result.reshape(shape), conf + + @staticmethod + def verify(cnp.ndarray src_data, cnp.ndarray dec_data) -> Tuple[float, float, float]: + """ + Compare decompressed data with original data. + + Parameters + ---------- + src_data : numpy.ndarray + Original data before compression + dec_data : numpy.ndarray + Decompressed data to verify + + Returns + ------- + max_diff : float + Maximum absolute difference + psnr : float + Peak Signal-to-Noise Ratio in dB + nrmse : float + Normalized Root Mean Square Error + """ + # Check shapes match by comparing size and ndim + if src_data.ndim != dec_data.ndim or src_data.size != dec_data.size: + raise ValueError("Shape mismatch between src_data and dec_data") + + if src_data.dtype != dec_data.dtype: + raise ValueError("Dtype mismatch between src_data and dec_data") + + # Integer differences and ranges can overflow the integer type. + if np.issubdtype(src_data.dtype, np.integer): + src_data = src_data.astype(np.float64) + dec_data = dec_data.astype(np.float64) + + # Calculate data range and difference + cdef double data_range = np.max(src_data) - np.min(src_data) + cdef cnp.ndarray diff = src_data - dec_data + cdef double max_diff = np.max(np.abs(diff)) + + # Calculate MSE and derived metrics + cdef double mse = np.mean(diff ** 2) + cdef double nrmse = np.sqrt(mse) / data_range if data_range > 0 else 0.0 + cdef double psnr = 20 * np.log10(data_range) - 10 * np.log10(mse) if mse > 0 else float('inf') + + return max_diff, psnr, nrmse diff --git a/tools/pysz/tests/__init__.py b/tools/pysz/tests/__init__.py new file mode 100644 index 00000000..a997a422 --- /dev/null +++ b/tools/pysz/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for pysz package.""" diff --git a/tools/pysz/tests/test_pysz.py b/tools/pysz/tests/test_pysz.py new file mode 100644 index 00000000..42081bf2 --- /dev/null +++ b/tools/pysz/tests/test_pysz.py @@ -0,0 +1,117 @@ +""" +Test script for pysz - Python interface for SZ3 compression + +Usage: + pytest tests/ # Run with pytest + python tests/test_pysz.py # Run standalone +""" + +import sys +import numpy as np + +from pysz import sz, szConfig, szErrorBoundMode, szAlgorithm + + +def test_compression(): + """Test basic compression and decompression""" + + print("=" * 70) + print("pysz Basic Test") + print("=" * 70) + + # Create test data + print("\n[1] Creating test data...") + data = np.random.randn(100, 100).astype(np.float32) + print(f" ✓ Shape: {data.shape}, dtype: {data.dtype}, size: {data.nbytes} bytes") + + # Create config + print("[2] Creating config...") + config = szConfig() + config.errorBoundMode = szErrorBoundMode.ABS + config.absErrorBound = 0.01 + print(f" ✓ Mode: {config.errorBoundMode}, Bound: {config.absErrorBound}") + + # Compress + print("[3] Compressing...") + compressed, ratio = sz.compress(data, config) + print(f" ✓ Ratio: {ratio:.2f}x ({data.nbytes} → {len(compressed)} bytes)") + + # Decompress + print("[4] Decompressing...") + decompressed, dec_config = sz.decompress(compressed, data.dtype, data.shape) + print(f" ✓ Shape: {decompressed.shape}") + + # Verify + print("[5] Verifying...") + max_error, psnr, nrmse = sz.verify(data, decompressed) + print(f" ✓ Max error: {max_error:.2e}, PSNR: {psnr:.2f} dB, NRMSE: {nrmse:.2e}") + + assert max_error <= 0.01, f"Error {max_error} exceeds bound 0.01" + + # Test double precision + print("[6] Testing double precision...") + data_double = np.random.randn(50, 50).astype(np.float64) + config_double = szConfig() + config_double.errorBoundMode = szErrorBoundMode.ABS + config_double.absErrorBound = 1e-6 + compressed_d, ratio_d = sz.compress(data_double, config_double) + decompressed_d, _ = sz.decompress(compressed_d, data_double.dtype, data_double.shape) + max_error_d, _, _ = sz.verify(data_double, decompressed_d) + print(f" ✓ Ratio: {ratio_d:.2f}x, Max error: {max_error_d:.2e}") + assert max_error_d <= 1e-6, f"Double error {max_error_d} exceeds bound 1e-6" + + # Test 3D data + print("[7] Testing 3D data...") + data_3d = np.random.randn(20, 30, 40).astype(np.float32) + config_3d = szConfig() + config_3d.errorBoundMode = szErrorBoundMode.REL + config_3d.relErrorBound = 0.001 + compressed_3d, ratio_3d = sz.compress(data_3d, config_3d) + decompressed_3d, _ = sz.decompress(compressed_3d, data_3d.dtype, data_3d.shape) + max_error_3d, _, _ = sz.verify(data_3d, decompressed_3d) + print(f" ✓ Ratio: {ratio_3d:.2f}x, Max error: {max_error_3d:.2e}") + + print("\n" + "=" * 70) + print("All tests passed!") + print("=" * 70) + + +def test_small_and_integer(): + """Small arrays and integer data fit the compression buffer""" + config = szConfig() + config.errorBoundMode = szErrorBoundMode.ABS + config.absErrorBound = 2 + for data in (np.arange(5, dtype=np.float32), + np.random.randint(-1000, 1000, size=(64, 64)).astype(np.int32), + np.random.randint(-2**40, 2**40, size=1000, dtype=np.int64)): + compressed, _ = sz.compress(data, config) + decompressed, _ = sz.decompress(compressed, data.dtype, data.shape) + max_error, _, _ = sz.verify(data, decompressed) + assert max_error <= 2, f"{data.dtype}: error {max_error} exceeds bound 2" + + +def test_wrong_shape(): + """A shape that does not match the compressed data is refused""" + data = np.random.rand(64, 64).astype(np.float32) + config = szConfig() + config.errorBoundMode = szErrorBoundMode.ABS + config.absErrorBound = 1e-3 + compressed, _ = sz.compress(data, config) + try: + sz.decompress(compressed, data.dtype, (8, 8)) + except ValueError: + return + raise AssertionError("decompress accepted a shape of 64 values for 4096") + + +if __name__ == '__main__': + try: + test_compression() + test_small_and_integer() + test_wrong_shape() + sys.exit(0) + except Exception as e: + print(f"\n✗ Test failed: {e}") + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/tools/sz3/CMakeLists.txt b/tools/sz3/CMakeLists.txt new file mode 100644 index 00000000..46d7210b --- /dev/null +++ b/tools/sz3/CMakeLists.txt @@ -0,0 +1,16 @@ +file(GLOB cpu_source_files "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +foreach (SRCFILE IN LISTS cpu_source_files) + get_filename_component(EXE ${SRCFILE} NAME_WLE) + message(STATUS "Target = ${EXE}") + add_executable(${EXE} ${EXE}.cpp) + target_link_libraries(${EXE} PRIVATE SZ3) + # Ask for the binary rather than spelling out where it lands: a multi-config generator puts it + # under a per-config subdirectory, and the install then cannot find it. + if (SZ3_INSTALL AND (EXE STREQUAL "sz3" OR EXE STREQUAL "sz3_smoke_test")) + install(PROGRAMS $ TYPE BIN) + endif () + +endforeach () + +#add_subdirectory(demo) diff --git a/tools/sz3/sz3.config b/tools/sz3/sz3.config new file mode 100644 index 00000000..fa24c855 --- /dev/null +++ b/tools/sz3/sz3.config @@ -0,0 +1,48 @@ +# SZ3 Configuration File +# +# This file configures SZ3 compression and decompression settings. +# Uncomment and modify the settings below as needed. +# For detailed explanations of each option and their valid values, refer to SZ3/utils/Config.hpp. +# +# Default values are used if settings are commented out or omitted. + +[GlobalSettings] + +# Compression algorithm +# Options: ALGO_LORENZO_REG, ALGO_INTERP_LORENZO, ALGO_INTERP, ALGO_NOPRED, ALGO_LOSSLESS, ALGO_BIOMD +CmprAlgo = ALGO_INTERP_LORENZO + +# Error bound mode +# Options: ABS, REL, PSNR, NORM, ABS_AND_REL, ABS_OR_REL +ErrorBoundMode = ABS + +# Absolute error bound (used with ABS mode) +AbsErrorBound = 1e-3 + +# Relative error bound (used with REL mode) +# RelErrorBound = 1e-3 + +# PSNR error bound (used with PSNR mode) +# PSNRErrorBound = 70 + +# L2 norm error bound (used with NORM mode) +# L2NormErrorBound = 0.333 + +# Enable OpenMP for parallel processing (YES or NO) +# OpenMP = NO + + + +[AlgoSettings] + +# Advanced algorithm settings for tuning (not recommended for regular users) +# Uncomment only if you understand the implications. + +# InterpolationAlgo = INTERP_ALGO_CUBIC +# InterpolationDirection = 0 +# Lorenzo = Yes +# Lorenzo2ndOrder = No +# Regression = Yes +# Regression2ndOrder = No +# BlockSize = 6 +# QuantizationBinTotal = 65536 diff --git a/tools/sz3/sz3.cpp b/tools/sz3/sz3.cpp new file mode 100644 index 00000000..2a9e1ae9 --- /dev/null +++ b/tools/sz3/sz3.cpp @@ -0,0 +1,542 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" + +#define SZ_FLOAT 0 +#define SZ_DOUBLE 1 +#define SZ_UINT8 2 +#define SZ_INT8 3 +#define SZ_UINT16 4 +#define SZ_INT16 5 +#define SZ_UINT32 6 +#define SZ_INT32 7 +#define SZ_UINT64 8 +#define SZ_INT64 9 + +inline void usage() { + printf("Note: SZ3 command line arguments are backward compatible with SZ2, \n"); + printf(" use -h2 to show the supported SZ2 command line arguments. \n"); + printf("Usage: sz3 \n"); + printf("Options:\n"); + printf("* general options:\n"); + printf(" -h: print the help information\n"); + printf(" -h2: print the help information for SZ2 style command line\n"); + printf(" -v: print the version number\n"); + printf(" -a : print compression results such as distortions\n"); + printf("* input and output:\n"); + printf(" -i : original input file in binary format\n"); + printf(" -o : decompressed file in binary format\n"); + printf(" -z : compressed file\n"); + printf(" -t : store decompressed file in text format\n"); + // printf(" -p: print meta data (configuration info)\n"); + printf("* data type:\n"); + printf(" -f: single precision (float type)\n"); + printf(" -d: double precision (double type)\n"); + printf(" -I : integer type (width = 32 or 64)\n"); + printf("* configuration file: \n"); + printf(" -c : configuration file sz.config\n"); + printf("* error control: (the error control parameters here will overwrite the setting in sz.config)\n"); + printf(" -M \n"); + printf(" error control mode as follows: \n"); + printf(" ABS (absolute error bound)\n"); + printf(" REL (value range based error bound, so a.k.a., VR_REL)\n"); + printf(" PSNR (peak signal-to-noise ratio)\n"); + printf(" NORM (norm2 error : sqrt(sum(xi-xi')^2)\n"); + printf(" ABS_AND_REL (using min{ABS, REL})\n"); + printf(" ABS_OR_REL (using max{ABS, REL})\n"); + printf( + " error bound can be set directly after the error control mode, or separately with the following " + "options:\n"); + printf(" -A : specifying absolute error bound\n"); + printf(" -R : specifying relative error bound\n"); + // printf(" -P : specifying point-wise relative error bound\n"); + printf(" -S : specifying PSNR\n"); + printf(" -N : specifying normErr\n"); + printf("* dimensions: \n"); + printf(" -1 : dimension for 1D data such as data[nx]\n"); + printf(" -2 : dimensions for 2D data such as data[ny][nx]\n"); + printf(" -3 : dimensions for 3D data such as data[nz][ny][nx] \n"); + printf(" -4 : dimensions for 4D data such as data[np][nz][ny][nx] \n"); + printf("* examples: \n"); + printf(" sz -f -i test.dat -z test.dat.sz -3 8 8 128 -M ABS 1e-3 \n"); + printf(" sz -f -z test.dat.sz -o test.dat.sz.out -3 8 8 128 -M REL 1e-3 -a \n"); + printf(" sz -f -i test.dat -o test.dat.sz.out -3 8 8 128 -M ABS_AND_REL -A 1 -R 1e-3 -a \n"); + printf(" sz -f -i test.dat -o test.dat.sz.out -3 8 8 128 -c sz.config \n"); + printf(" sz -f -i test.dat -o test.dat.sz.out -3 8 8 128 -c sz.config -M ABS 1e-3 -a\n"); + exit(0); +} + +inline void usage_sz2() { + printf("Note: below are the supported command line arguments in SZ2 style\n"); + printf("Usage: sz \n"); + printf("Options:\n"); + printf("* operation type:\n"); + printf(" -z : the compression operation with an optionally specified output file.\n"); + printf(" (the compressed file will be named as .sz if not specified)\n"); + printf(" -x : the decompression operation with an optionally specified output file\n"); + printf(" (the decompressed file will be named as .out if not specified)\n"); + printf(" -p: print meta data (configuration info)\n"); + printf(" -h: print the help information\n"); + printf(" -v: print the version number\n"); + printf("* data type:\n"); + printf(" -f: single precision (float type)\n"); + printf(" -d: double precision (double type)\n"); + printf("* configuration file: \n"); + printf(" -c : configuration file sz.config\n"); + printf("* error control: (the error control parameters here will overwrite the setting in sz.config)\n"); + printf(" -M : 10 options as follows. \n"); + printf(" ABS (absolute error bound)\n"); + printf(" REL (value range based error bound, so a.k.a., VR_REL)\n"); + printf(" ABS_AND_REL (using min{ABS, REL})\n"); + printf(" ABS_OR_REL (using max{ABS, REL})\n"); + printf(" PSNR (peak signal-to-noise ratio)\n"); + printf(" NORM (norm2 error : sqrt(sum(xi-xi')^2)\n"); + // printf(" PW_REL (point-wise relative error bound)\n"); + printf(" -A : specifying absolute error bound\n"); + printf(" -R : specifying relative error bound\n"); + // printf(" -P : specifying point-wise relative error bound\n"); + printf(" -S : specifying PSNR\n"); + printf(" -N : specifying normErr\n"); + printf("* input data file:\n"); + printf(" -i : original data file\n"); + printf(" -s : compressed data file in decompression\n"); + printf("* output type of decompressed file: \n"); + printf(" -b (by default) : decompressed file stored in binary format\n"); + printf(" -t : decompreadded file stored in text format\n"); + // printf(" -T : pre-processing with Tucker Tensor Decomposition\n"); + printf("* dimensions: \n"); + printf(" -1 : dimension for 1D data such as data[nx]\n"); + printf(" -2 : dimensions for 2D data such as data[ny][nx]\n"); + printf(" -3 : dimensions for 3D data such as data[nz][ny][nx] \n"); + printf(" -4 : dimensions for 4D data such as data[np][nz][ny][nx] \n"); + printf("* print compression results: \n"); + printf(" -a : print compression results such as distortions\n"); + printf("* examples: \n"); + printf(" sz -z -f -c sz.config -i testdata/x86/testfloat_8_8_128.dat -3 8 8 128\n"); + printf(" sz -z -f -c sz.config -M ABS -A 1E-3 -i testdata/x86/testfloat_8_8_128.dat -3 8 8 128\n"); + printf(" sz -x -f -s testdata/x86/testfloat_8_8_128.dat.sz -3 8 8 128\n"); + printf( + " sz -x -f -s testdata/x86/testfloat_8_8_128.dat.sz -i testdata/x86/testfloat_8_8_128.dat -3 8 8 128 " + "-a\n"); + printf(" sz -z -d -c sz.config -i testdata/x86/testdouble_8_8_128.dat -3 8 8 128\n"); + printf(" sz -x -d -s testdata/x86/testdouble_8_8_128.dat.sz -3 8 8 128\n"); + printf(" sz -p -s testdata/x86/testdouble_8_8_128.dat.sz\n"); + exit(0); +} + +// A malformed argument is a caller error, not a help request: report it and let main() exit non-zero. +[[noreturn]] static void argError(const std::string &msg) { throw std::invalid_argument(msg); } + +// Every SZ3 size downstream is derived from this, so nothing corrupt or absurd may get past here. +static size_t parse_dim(const char *s) { + if (s[0] < '0' || s[0] > '9') argError(std::string("invalid dimension '") + s + "'"); + errno = 0; + char *end = nullptr; + unsigned long long v = strtoull(s, &end, 10); + bool overflows_size_t = false; + if constexpr (sizeof(size_t) < sizeof(unsigned long long)) { + overflows_size_t = v > static_cast(SIZE_MAX); + } + if (*end != '\0' || errno == ERANGE || v == 0 || overflows_size_t) + argError(std::string("invalid dimension '") + s + "'"); + return static_cast(v); +} + +template +void compress(char *inPath, char *cmpPath, SZ3::Config &conf) { + T *data = new T[conf.num]; + SZ3::readfile(inPath, conf.num, data); + // SZ_compress refuses a capacity below its own bound, which a small input falls under. + size_t bytesCap = SZ3::SZ_compress_size_bound(conf); + auto bytes = new char[bytesCap]; + + SZ3::Timer timer(true); + size_t outSize = SZ_compress(conf, data, bytes, bytesCap); + double compress_time = timer.stop(); + + char outputFilePath[1024]; + if (cmpPath == nullptr) { + snprintf(outputFilePath, 1024, "%s.sz", inPath); + } else { + strcpy(outputFilePath, cmpPath); + } + SZ3::writefile(outputFilePath, bytes, outSize); + + printf("compression ratio = %.2f \n", conf.num * 1.0 * sizeof(T) / outSize); + printf("compression time = %f\n", compress_time); + printf("compressed data file = %s\n", outputFilePath); + + delete[] data; + delete[] bytes; +} + +template +void decompress(char *inPath, char *cmpPath, char *decPath, SZ3::Config &conf, int binaryOutput, int printCmpResults) { + size_t cmpSize; + auto cmpData = SZ3::readfile(cmpPath, cmpSize); + + SZ3::Timer timer(true); + auto decData = SZ_decompress(conf, cmpData.get(), cmpSize); + double compress_time = timer.stop(); + + char outputFilePath[1024]; + if (decPath == nullptr) { + snprintf(outputFilePath, 1024, "%s.out", cmpPath); + } else { + strcpy(outputFilePath, decPath); + } + if (binaryOutput == 1) { + SZ3::writefile(outputFilePath, decData, conf.num); + } else { + SZ3::writeTextFile(outputFilePath, decData, conf.num); + } + if (printCmpResults) { + // compute the distortion / compression errors... + size_t totalNbEle; + auto ori_data = SZ3::readfile(inPath, totalNbEle); + assert(totalNbEle == conf.num); + SZ3::verify(ori_data.get(), decData, conf.num); + } + delete[] decData; + + printf("compression ratio = %f\n", conf.num * sizeof(T) * 1.0 / cmpSize); + printf("decompression time = %f seconds.\n", compress_time); + printf("decompressed file = %s\n", outputFilePath); +} + +static int run(int argc, char *argv[]) { + bool binaryOutput = true; + int printCmpResults = 0; + int printMeta = 0; + bool compression = false; + bool decompression = false; + int dataType = SZ_FLOAT; + char *inPath = nullptr; + char *cmpPath = nullptr; + char *conPath = nullptr; + char *decPath = nullptr; + bool delCmpPath = false; + + char *errBoundMode = nullptr; + char *errBound = nullptr; + char *absErrorBound = nullptr; + char *relErrorBound = nullptr; + // char *pwrErrorBound = nullptr; + char *psnrErrorBound = nullptr; + char *normErrorBound = nullptr; + + bool sz2mode = false; + + size_t r4 = 0; + size_t r3 = 0; + size_t r2 = 0; + size_t r1 = 0; + + int i = 0; + // int status; + if (argc == 1) usage(); + int width = -1; + + for (i = 1; i < argc; i++) { + if (argv[i][0] != '-' || argv[i][2]) { + if (argv[i][1] == 'h' && argv[i][2] == '2') { + usage_sz2(); + } else if (strcmp(argv[i], "--help") == 0) { + usage(); + } else { + argError(std::string("unrecognized argument: ") + argv[i]); + } + } + switch (argv[i][1]) { + case 'h': + usage(); + exit(0); + case 'v': + printf("SZ3 Version: %s\n", SZ3_VER); + printf("SZ3 Data Format Version: %s\n", SZ3_DATA_VER); + exit(0); + case 'b': + binaryOutput = true; + break; + case 't': + binaryOutput = false; + break; + case 'a': + printCmpResults = 1; + break; + case 'p': + printMeta = 1; + break; + case 'z': + compression = true; + if (i + 1 < argc) { + cmpPath = argv[i + 1]; + if (cmpPath[0] != '-') + i++; + else + cmpPath = nullptr; + } + break; + case 'x': + sz2mode = true; + decompression = true; + if (i + 1 < argc) { + decPath = argv[i + 1]; + if (decPath[0] != '-') + i++; + else + decPath = nullptr; + } + break; + case 'f': + dataType = SZ_FLOAT; + break; + case 'd': + dataType = SZ_DOUBLE; + break; + case 'I': + if (++i == argc || sscanf(argv[i], "%d", &width) != 1) { + argError("-I requires an integer width (32 or 64)"); + } + if (width == 32) { + dataType = SZ_INT32; + } else if (width == 64) { + dataType = SZ_INT64; + } else { + argError("-I width must be 32 or 64"); + } + break; + case 'i': + if (++i == argc) argError("-i requires a file path"); + inPath = argv[i]; + break; + case 'o': + if (++i == argc) argError("-o requires a file path"); + decPath = argv[i]; + break; + case 's': + sz2mode = true; + if (++i == argc) argError("-s requires a file path"); + cmpPath = argv[i]; + break; + case 'c': + if (++i == argc) argError("-c requires a config file path"); + conPath = argv[i]; + break; + case '1': + if (++i == argc) argError("-1 requires 1 dimension"); + r1 = parse_dim(argv[i]); + break; + case '2': + if (++i == argc) argError("-2 requires 2 dimensions"); + r1 = parse_dim(argv[i]); + if (++i == argc) argError("-2 requires 2 dimensions"); + r2 = parse_dim(argv[i]); + break; + case '3': + if (++i == argc) argError("-3 requires 3 dimensions"); + r1 = parse_dim(argv[i]); + if (++i == argc) argError("-3 requires 3 dimensions"); + r2 = parse_dim(argv[i]); + if (++i == argc) argError("-3 requires 3 dimensions"); + r3 = parse_dim(argv[i]); + break; + case '4': + if (++i == argc) argError("-4 requires 4 dimensions"); + r1 = parse_dim(argv[i]); + if (++i == argc) argError("-4 requires 4 dimensions"); + r2 = parse_dim(argv[i]); + if (++i == argc) argError("-4 requires 4 dimensions"); + r3 = parse_dim(argv[i]); + if (++i == argc) argError("-4 requires 4 dimensions"); + r4 = parse_dim(argv[i]); + break; + case 'M': { + if (++i == argc) argError("-M requires an error bound mode"); + errBoundMode = argv[i]; + // match_enum silently keeps the default when the name is unknown, which would accept a typo as + // ABS. VR_REL is an alias resolved during setup, so it is not in EB_MAP. + bool knownMode = strcmp(errBoundMode, "VR_REL") == 0; + for (const auto &kv : SZ3::EB_MAP) { + if (SZ3::to_lower(kv.first) == SZ3::to_lower(errBoundMode)) knownMode = true; + } + if (!knownMode) argError(std::string("unknown error bound mode: ") + errBoundMode); + if (i + 1 < argc && argv[i + 1][0] != '-') { + errBound = argv[++i]; + } + break; + } + case 'A': + if (++i == argc) argError("-A requires an absolute error bound"); + absErrorBound = argv[i]; + break; + case 'R': + if (++i == argc) argError("-R requires a relative error bound"); + relErrorBound = argv[i]; + break; + // case 'P': + // if (++i == argc) + // usage(); + // pwrErrorBound = argv[i]; + // break; + case 'N': + if (++i == argc) argError("-N requires a norm error bound"); + normErrorBound = argv[i]; + break; + case 'S': + if (++i == argc) argError("-S requires a PSNR value"); + psnrErrorBound = argv[i]; + break; + default: + argError(std::string("unknown option: ") + argv[i]); + break; + } + } + + if ((inPath == nullptr) && (cmpPath == nullptr)) { + argError("you need to specify either a raw binary data file (-i) or a compressed data file (-s/-z) as input"); + } + + if (!sz2mode && inPath != nullptr && cmpPath != nullptr) { + compression = true; + } + if (cmpPath != nullptr && decPath != nullptr) { + decompression = true; + } + char cmpPathTmp[1024]; + if (inPath != nullptr && cmpPath == nullptr && decPath != nullptr) { + compression = true; + decompression = true; + snprintf(cmpPathTmp, 1024, "%s.sz.tmp", inPath); + cmpPath = cmpPathTmp; + delCmpPath = true; + } + if (inPath == nullptr || (errBoundMode == nullptr && conPath == nullptr)) { + compression = false; + } + if (!compression && !decompression) { + argError("nothing to do: specify compression (-i with -z/-o and an error bound) or decompression (-s/-x)"); + } + + SZ3::Config conf; + if (r2 == 0) { + conf = SZ3::Config(r1); + } else if (r3 == 0) { + conf = SZ3::Config(r2, r1); + } else if (r4 == 0) { + conf = SZ3::Config(r3, r2, r1); + } else { + conf = SZ3::Config(r4, r3, r2, r1); + } + if (compression && conPath != nullptr) { + conf.loadcfg(conPath); + } + + if (errBoundMode != nullptr) { + { + // backward compatible with SZ2 + if (relErrorBound != nullptr) { + conf.relErrorBound = atof(relErrorBound); + } + if (absErrorBound != nullptr) { + conf.absErrorBound = atof(absErrorBound); + } + if (psnrErrorBound != nullptr) { + conf.psnrErrorBound = atof(psnrErrorBound); + } + if (normErrorBound != nullptr) { + conf.l2normErrorBound = atof(normErrorBound); + } + } + + SZ3::match_enum(errBoundMode, SZ3::EB_MAP, conf.errorBoundMode); + if (strcmp(errBoundMode, "VR_REL") == 0) { + conf.errorBoundMode = SZ3::EB_REL; + } + if (conf.errorBoundMode == SZ3::EB_ABS) { + if (errBound != nullptr) { + conf.absErrorBound = atof(errBound); + } + } else if (conf.errorBoundMode == SZ3::EB_REL) { + if (errBound != nullptr) { + conf.relErrorBound = atof(errBound); + } + } else if (conf.errorBoundMode == SZ3::EB_PSNR) { + if (errBound != nullptr) { + conf.psnrErrorBound = atof(errBound); + } + } else if (conf.errorBoundMode == SZ3::EB_L2NORM) { + if (errBound != nullptr) { + conf.l2normErrorBound = atof(errBound); + } + } else if (conf.errorBoundMode == SZ3::EB_ABS_AND_REL) { + } else if (conf.errorBoundMode == SZ3::EB_ABS_OR_REL) { + } else { + argError("wrong error bound mode setting by using the option '-M'"); + } + } + + if (compression) { + if (dataType == SZ_FLOAT) { + compress(inPath, cmpPath, conf); +#if (!SZ3_DEBUG_TIMINGS) + } else if (dataType == SZ_DOUBLE) { + compress(inPath, cmpPath, conf); + } else if (dataType == SZ_INT32) { + compress(inPath, cmpPath, conf); + } else if (dataType == SZ_INT64) { + compress(inPath, cmpPath, conf); +#endif + } else { + argError("data type not supported"); + } + } + if (decompression) { + if (printCmpResults && inPath == nullptr) { + argError("the -a option (analysis) needs the original data path, specify it with -i "); + } + + if (dataType == SZ_FLOAT) { + decompress(inPath, cmpPath, decPath, conf, binaryOutput, printCmpResults); +#if (!SZ3_DEBUG_TIMINGS) + } else if (dataType == SZ_DOUBLE) { + decompress(inPath, cmpPath, decPath, conf, binaryOutput, printCmpResults); + } else if (dataType == SZ_INT32) { + decompress(inPath, cmpPath, decPath, conf, binaryOutput, printCmpResults); + } else if (dataType == SZ_INT64) { + decompress(inPath, cmpPath, decPath, conf, binaryOutput, printCmpResults); +#endif + } else { + argError("data type not supported"); + } + } + if (printMeta) { + conf.print(); + } + if (delCmpPath) { + remove(cmpPath); + } + return 0; +} + +// A configuration the algorithms refuse arrives here as an exception; a caller needs a message +// and an exit code, not a signal. +int main(int argc, char *argv[]) { + try { + return run(argc, argv); + } catch (const std::exception &e) { + std::cerr << "sz3: " << e.what() << std::endl; + } catch (...) { + std::cerr << "sz3: unknown error" << std::endl; + } + return 1; +} diff --git a/tools/sz3/sz3_customized_demo.cpp b/tools/sz3/sz3_customized_demo.cpp new file mode 100644 index 00000000..971eb32a --- /dev/null +++ b/tools/sz3/sz3_customized_demo.cpp @@ -0,0 +1,181 @@ +/** + * This is a demo for creating your own compressor in SZ3. + * Essentially, there are two main steps: first, you need to implement your own compressor. + * Please check out the 4 examples provided in the main function for implementing your own compressor. + * + * Second, you need to run your compressor with SZ3 executable or a new executable. + * This demon code is a new executable, so you have control over the parameter parsing, IO, etc. + * If you want to use the SZ3 executable to run your own compressor, please follow the steps below: + * 1. Add a new ALGO in SZ3 Config + * 2. Add a new hpp file in "include/SZ3/api/impl/" to assemble your compressor, one example easy to follow is + * "include/SZ3/api/impl/SZAlgoNopred.hpp" + * 3. Add the corresponding code in "include/SZ3/api/impl/SZDispatcher.hpp" to dispatch the new compressor. + * 4. When executing the SZ3 executable, use -c to specify the config file. The config file should use the new ALGO. + * Example config file is "tools/sz3/sz3.config". + */ + +#include "SZ3/api/sz.hpp" +using namespace SZ3; + +template +void SZ3_interpolation_compress(Config &conf, T *data, char *dst, size_t &outSize) { + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), Lossless_zstd()); + sz->compress(conf, data, reinterpret_cast(dst), outSize); +} + +template +void SZ3_interpolation_decompress(const Config &conf, const char *cmpData, size_t cmpSize, T *decData) { + auto cmpDataPos = reinterpret_cast(cmpData); + auto sz = make_compressor_sz_generic( + make_decomposition_interpolation(conf, LinearQuantizer(conf.absErrorBound, conf.quantbinCnt / 2)), + HuffmanEncoder(), Lossless_zstd()); + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + +template +class MyDecomposition : public concepts::DecompositionInterface { + public: + MyDecomposition(const Config &conf) : num(conf.num), quantizer(conf.absErrorBound) {} + + std::vector compress(const Config &conf, T *data) override { + // write your own logic here + std::vector output(num); + for (size_t i = 0; i < num; i++) { + output[i] = quantizer.quantize_and_overwrite(data[i], 0); // this will replace the value of data[i] + } + return output; + } + + T *decompress(const Config &conf, std::vector &quant_inds, T *dec_data) override { + // write your own logic here + for (size_t i = 0; i < num; i++) { + dec_data[i] = quantizer.recover(0, quant_inds[i]); + } + return dec_data; + } + + void save(uchar *&c) override { + write(num, c); + quantizer.save(c); + } + + void load(const uchar *&c, size_t &remaining_length) override { + uint64_t num64 = 0; + read(num64, c, remaining_length); + num = num64; + quantizer.load(c, remaining_length); + } + + std::pair get_out_range() override { return std::make_pair(0, 0); } + + private: + size_t num; + LinearQuantizer quantizer; +}; + +template +void SZ3_customized_compress(Config &conf, T *data, char *dst, size_t &outSize) { + calAbsErrorBound(conf, data); + + auto sz = make_compressor_sz_generic(MyDecomposition(conf), HuffmanEncoder(), Lossless_zstd()); + + outSize = sz->compress(conf, data, reinterpret_cast(dst), outSize); +} + +template +void SZ3_customized_decompress(const Config &conf, const char *cmpData, size_t cmpSize, T *decData) { + auto cmpDataPos = reinterpret_cast(cmpData); + auto sz = make_compressor_sz_generic(MyDecomposition(conf), HuffmanEncoder(), Lossless_zstd()); + + sz->decompress(conf, cmpDataPos, cmpSize, decData); +} + +int main(int argc, char **argv) { + if (argc < 2) { + std::cout << "SZ v" << SZ3_VER << std::endl; + std::cout << "usage: " << argv[0] << " data_file -num_dim dim0 .. dimn ABS" << std::endl; + std::cout << "example: " << argv[0] << " qmcpack.dat -3 33120 69 69 1e-3" << std::endl; + return 0; + } + + std::string src_file_name = argv[1]; + + // read dimensions + int dim = atoi(argv[2] + 1); + assert(1 <= dim && dim <= 4); + if (dim != 3) { + printf("This demo only supports 3D data.\n"); + return 0; + } + int argp = 3; + std::vector dims(dim); + for (int i = 0; i < dim; i++) { + dims[i] = atoi(argv[argp++]); + } + + // create config with data dimensions + Config conf; + conf.setDims(dims.begin(), dims.end()); + + // prepare input and output buffers + std::vector input_data(conf.num); + SZ3::readfile(argv[1], conf.num, input_data.data()); + std::vector input_data_copy(input_data); + std::vector dec_data(conf.num); + auto dec_data_pos = dec_data.data(); + std::vector cmpData(conf.num * sizeof(float) * 2); + size_t cmpSize = cmpData.size(); + + // set error bound (you can also use relative error bound EB_REL etc.) + conf.errorBoundMode = EB_ABS; + conf.absErrorBound = atof(argv[argp++]); + + // ================================================================================ + // There are 4 customization examples, you just need to pick up one. + + // Example 1: using SZ3 API defined in SZ3/API/sz.hpp + // SZ3 has some built-in compressors with fixed modules, for example, the interpolation compressor uses + // interpolation decomposition, linear quantizer, and huffman encoder. + // The API takes the SZ3::Config as parameter which allows some basic customization, such as interpolation + // method (linear or cubic) and the quantization levels. + SZ_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ_decompress(conf, cmpData.data(), cmpSize, dec_data_pos); + + // Example 2: using provided modules to assemble a generic compressor + // The generic compressor follow the decomposition -> encoding -> lossless pipeline. + // You can use any decomposition, encoder, and lossless modules to form such a compressor, including bypass + // modules. This example assembles the interpolation compressor with interpolation decomposition, linear + // quantizer, and huffman encoder. You can change any of the modules to get a new compressor that is not + // built-in in SZ3. + SZ3_interpolation_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ3_interpolation_decompress(conf, cmpData.data(), cmpSize, dec_data_pos); + + // Example 3: using new modules to assemble a generic compressor + // This is an extension of Example 2, which uses a custom decomposition module instead of built-in ones. + // MyDecomposition is the new decomposition module, which implements the DecompositionInterface. + // We put the source code of MyDecomposition here for demonstration purpose. + // You should put your new modules in the corresponding module folders (include/SZ3/decomposition for Decomposition + // modules). + SZ3_customized_compress(conf, input_data.data(), cmpData.data(), cmpSize); + SZ3_customized_decompress(conf, cmpData.data(), cmpSize, dec_data_pos); + + // Example 4: assemble a specialized compressor + // If your compressor doesn't follow the decomposition -> encoding -> lossless pipeline, + // you can implement your own compressor by implementing the CompressorInterface + // Please check out "SZ3/compressor/specialized/SZExaaltCompressor.hpp". It contains two separate encoding process. + + // ================================================================================ + + // In the end, you can use the SZ3 API to verify the decompressed data. + SZ3::verify(input_data_copy.data(), dec_data_pos, conf.num); + + // write compressed data and decompressed data to the same folder as original data + writefile((src_file_name + ".demo").data(), cmpData.data(), cmpSize); + writefile((src_file_name + ".demo.out").data(), dec_data.data(), conf.num); + + return 0; +} diff --git a/tools/sz3/sz3_smoke_test.cpp b/tools/sz3/sz3_smoke_test.cpp new file mode 100644 index 00000000..a6c7b29d --- /dev/null +++ b/tools/sz3/sz3_smoke_test.cpp @@ -0,0 +1,50 @@ +// +// Created by Kai Zhao on 11/10/22. +// + +#include + +#include "SZ3/encoder/BypassEncoder.hpp" +#include "SZ3/lossless/Lossless_bypass.hpp" + +int main(int argc, char **argv) { + std::vector dims({100, 200, 300}); + SZ3::Config conf({dims[0], dims[1], dims[2]}); + conf.cmprAlgo = SZ3::ALGO_INTERP_LORENZO; + conf.errorBoundMode = SZ3::EB_ABS; // refer to def.hpp for all supported error bound mode + conf.absErrorBound = 1E-3; // absolute error bound 1e-3 + + std::vector input_data(conf.num); + std::vector dec_data(conf.num); + std::vector stride({dims[1] * dims[2], dims[2], 1}); + + for (size_t i = 0; i < dims[0]; ++i) { + for (size_t j = 0; j < dims[1]; ++j) { + for (size_t k = 0; k < dims[2]; ++k) { + double x = static_cast(i) - static_cast(dims[0]) / 2.0; + double y = static_cast(j) - static_cast(dims[1]) / 2.0; + double z = static_cast(k) - static_cast(dims[2]) / 2.0; + input_data[i * stride[0] + j * stride[1] + k] = + static_cast(.0001 * y * sin(y) + .0005 * cos(pow(x, 2) + x) + z); + } + } + } + + std::vector input_data_copy(input_data); + conf.openmp = true; + size_t cmpSize; + char *cmpData = SZ_compress(conf, input_data.data(), cmpSize); + auto dec_data_p = dec_data.data(); + SZ_decompress(conf, cmpData, cmpSize, dec_data_p); + + double max_err = 0.0; + for (size_t i = 0; i < conf.num; i++) { + if (fabs(dec_data[i] - input_data_copy[i]) > max_err) { + max_err = fabs(dec_data[i] - input_data_copy[i]); + } + } + printf("Smoke test %s", max_err <= conf.absErrorBound ? "passed" : "failed"); + // printf("%lu ", conf.num); + delete[] cmpData; + return 0; +} diff --git a/tools/sz3/testfloat_8_8_128.dat b/tools/sz3/testfloat_8_8_128.dat new file mode 100644 index 0000000000000000000000000000000000000000..1f9d40d84b5d0201197385c0cb090552d0191129 GIT binary patch literal 32768 zcmeI#Nl2AZ0L5{Mv=npkf=Gf2tqhwjvWasqCItm0E=^yVW2W!>G%2G3El7%5n2=f_ zl}6B_f)I^HNhF$$Y%r;8q#}t1iY%#!Z}~X;^ICjd9%uEx_icVvQ(a7)kr^)hqg~AY<{-Am!O5!*4zxR1amj&gbD+K_9CdKkbuep+ zgY?${>^vU8woL)7S{A@$UjXWR^|cb9tOWbN6rNVit=1E)uRE!sQW9G5BJPnSN=d>D#x%?7q-w)c4QRT217d8>TVknMu$7YOcH; zYHR1l*!)+c?6=Z!R(((Xl3=59XWHgH^X*{&VmtC!nqA(x+9sE}c1ihmtG>5>tg$UU z4YsMb-j**cu$gtYOv%+yExK*od5STXGb z;^N+5!<9aKuk1zgr~6RfTPJtnK;3o3S9GGKyB!a6E}-OI8%lc5pegzk)c5S?CvYz9 zINoFi-b>Fi-b>Fi-b>Fi-b>Fi-b>Fi-b>Fi- zb>Fi-b>Fi-b>Fi-&$pp|s`}o0*I<3>zGr>vzGr>vzGr>vzGr>R$Lc%tvHH$@tiCfJ ztMAOm>O1qX`p$f;zB3=I@65;QJM*#n?)fm(&tN|G?nQNHK33nEkJWePWA&Z+SbbOb zJ?m5VJ?m5VJ?m5VJ?m5VJ?m5VJ?m5VJ?m5VJ?m5VJ?m5VJ?ry)8|tU3@4a^o)~D`! z)~D`!)~D`!)~D`!*2jFTzB3=I@65;QJM*#n&U~!CGasw(%*X0G^RfEQe5}4RAFJ=4 z4@3P7=40<(RCnfM^_}@xeP=#a-b_@v>b_@v>b_@v>b_@v%*X0G^RfEQe5}4R zAFJ=o$Lc%tvHH$@tiCfJtMAOm>O1qX`tJEK)X!i(_U=V>XFgWnnUB?X=417p`B;5d z_dV-V_dV-V_dV-V_dV-V_dV-V_dV-V_dV-V_dV-V_dV-V_dViVgs_(sb4c4db Td)BA!d)BA!d)BA!d)D_G>40MX literal 0 HcmV?d00001 diff --git a/tools/sz3c/CMakeLists.txt b/tools/sz3c/CMakeLists.txt new file mode 100644 index 00000000..ca419d8a --- /dev/null +++ b/tools/sz3c/CMakeLists.txt @@ -0,0 +1,30 @@ +add_library( + SZ3c + src/sz3c.cpp +) + +target_link_libraries( + SZ3c + PRIVATE SZ3 +) + +target_compile_definitions(SZ3c PRIVATE SZ3C_EXPORTS) +set_target_properties(SZ3c PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) +set_target_properties(SZ3c PROPERTIES VERSION ${SZ3_VERSION} SOVERSION ${SZ3_VERSION_MAJOR}.${SZ3_VERSION_MINOR}) + +target_include_directories( + SZ3c + PUBLIC + "$" + "$" +) + +if (SZ3_INSTALL) + install(TARGETS SZ3c EXPORT SZ3c + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ) + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/SZ3c) + install(EXPORT SZ3c NAMESPACE SZ3:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/SZ3) +endif () diff --git a/tools/sz3c/include/sz3c.h b/tools/sz3c/include/sz3c.h new file mode 100644 index 00000000..bf8e34a7 --- /dev/null +++ b/tools/sz3c/include/sz3c.h @@ -0,0 +1,64 @@ +// +// Created by Kai Zhao on 10/27/22. +// + +#ifndef SZ3_SZ3C_H +#define SZ3_SZ3C_H + +#include +/** Begin errorbound mode in SZ2 (defines.h) **/ +#define ABS 0 +#define REL 1 +#define VR_REL 1 // alternative name to REL +#define ABS_AND_REL 2 +#define ABS_OR_REL 3 +#define PSNR 4 +#define NORM 5 + +#define PW_REL 10 +#define ABS_AND_PW_REL 11 +#define ABS_OR_PW_REL 12 +#define REL_AND_PW_REL 13 +#define REL_OR_PW_REL 14 +/** End errorbound mode in SZ2 (defines.h) **/ + +/** Begin dataType in SZ2 (defines.h) **/ +#define SZ_FLOAT 0 +#define SZ_DOUBLE 1 +#define SZ_UINT8 2 +#define SZ_INT8 3 +#define SZ_UINT16 4 +#define SZ_INT16 5 +#define SZ_UINT32 6 +#define SZ_INT32 7 +#define SZ_UINT64 8 +#define SZ_INT64 9 +/** End dataType in SZ2 (defines.h) **/ + +#ifdef _WIN32 +#ifdef SZ3C_EXPORTS +#define SZ3C_API __declspec(dllexport) +#else +#define SZ3C_API __declspec(dllimport) +#endif +#else +#define SZ3C_API __attribute__((visibility("default"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif +/* Both return NULL on error, with a message on stderr. Free their results with free_buf(). */ +SZ3C_API unsigned char *SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, + double absErrBound, double relBoundRatio, double pwrBoundRatio, size_t r5, + size_t r4, size_t r3, size_t r2, size_t r1); + +SZ3C_API void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, + size_t r2, size_t r1); + +SZ3C_API void free_buf(void *p); +#ifdef __cplusplus +} +#endif + +#endif // SZ3_SZ3C_H diff --git a/tools/sz3c/src/sz3c.cpp b/tools/sz3c/src/sz3c.cpp new file mode 100644 index 00000000..1c7948d6 --- /dev/null +++ b/tools/sz3c/src/sz3c.cpp @@ -0,0 +1,118 @@ +// +// Created by Kai Zhao on 10/27/22. +// + +#include "sz3c.h" + +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" + +using namespace SZ3; + +// Errors return NULL with a message on stderr: a library must not end its caller's process, and an exception must not +// cross into C. +unsigned char *SZ_compress_args(int dataType, void *data, size_t *outSize, int errBoundMode, double absErrBound, + double relBoundRatio, double pwrBoundRatio, size_t r5, size_t r4, size_t r3, size_t r2, + size_t r1) try { + SZ3::Config conf; + if (r2 == 0) { + conf = SZ3::Config(r1); + } else if (r3 == 0) { + conf = SZ3::Config(r2, r1); + } else if (r4 == 0) { + conf = SZ3::Config(r3, r2, r1); + } else if (r5 == 0) { + conf = SZ3::Config(r4, r3, r2, r1); + } else { + conf = SZ3::Config(r5 * r4, r3, r2, r1); + } + // conf.loadcfg(conPath); + conf.absErrorBound = absErrBound; + conf.relErrorBound = relBoundRatio; + // conf.pwrErrorBound = pwrBoundRatio; + if (errBoundMode == ABS) { + conf.errorBoundMode = EB_ABS; + } else if (errBoundMode == REL) { + conf.errorBoundMode = EB_REL; + } else if (errBoundMode == ABS_AND_REL) { + conf.errorBoundMode = EB_ABS_AND_REL; + } else if (errBoundMode == ABS_OR_REL) { + conf.errorBoundMode = EB_ABS_OR_REL; + } else { + fprintf(stderr, "SZ3: errBoundMode %d not supported\n", errBoundMode); + return nullptr; + } + + unsigned char *cmpr_data = nullptr; + if (dataType == SZ_FLOAT) { + cmpr_data = reinterpret_cast(SZ_compress(conf, static_cast(data), *outSize)); +#if (!SZ3_DEBUG_TIMINGS) + } else if (dataType == SZ_DOUBLE) { + cmpr_data = reinterpret_cast(SZ_compress(conf, static_cast(data), *outSize)); +#endif + } else { + fprintf(stderr, "SZ3: dataType %d not supported\n", dataType); + return nullptr; + } + + // convert c++ memory (by 'new' operator) to c memory (by malloc) + auto *cmpr = static_cast(malloc(*outSize)); + if (cmpr) memcpy(cmpr, cmpr_data, *outSize); + delete[] cmpr_data; + + return cmpr; +} catch (const std::exception &e) { + fprintf(stderr, "SZ3: %s\n", e.what()); + return nullptr; +} + +// Decompresses into SZ3's own buffer first: the caller's dimensions are only checked against the stream once it is +// read. +template +static void *decompress(unsigned char *bytes, size_t byteLength, size_t n) { + SZ3::Config conf; + std::unique_ptr dec(SZ_decompress(conf, reinterpret_cast(bytes), byteLength)); + if (conf.num != n) { + fprintf(stderr, "SZ3: the data holds %zu values, not the %zu the dimensions give\n", conf.num, n); + return nullptr; + } + auto *out = static_cast(malloc(n * sizeof(T))); + if (out) memcpy(out, dec.get(), n * sizeof(T)); + return out; +} + +void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_t r5, size_t r4, size_t r3, size_t r2, + size_t r1) try { + size_t n = 0; + if (r2 == 0) { + n = r1; + } else if (r3 == 0) { + n = r1 * r2; + } else if (r4 == 0) { + n = r1 * r2 * r3; + } else if (r5 == 0) { + n = r1 * r2 * r3 * r4; + } else { + n = r1 * r2 * r3 * r4 * r5; + } + + if (dataType == SZ_FLOAT) { + return decompress(bytes, byteLength, n); +#if (!SZ3_DEBUG_TIMINGS) + } else if (dataType == SZ_DOUBLE) { + return decompress(bytes, byteLength, n); +#endif + } else { + fprintf(stderr, "SZ3: dataType %d not supported\n", dataType); + return nullptr; + } +} catch (const std::exception &e) { + fprintf(stderr, "SZ3: %s\n", e.what()); + return nullptr; +} + +void free_buf(void *p) { free(p); } diff --git a/tools/test/CMakeLists.txt b/tools/test/CMakeLists.txt new file mode 100644 index 00000000..6e5ec612 --- /dev/null +++ b/tools/test/CMakeLists.txt @@ -0,0 +1,89 @@ +enable_testing() +# Try to find GTest locally +find_package(GTest QUIET) +if(NOT GTest_FOUND) + message(STATUS "GTest not found locally; fetching from GitHub...") + include(FetchContent) + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.16.0 + ) + if (MSVC) + set(BUILD_SHARED_LIBS OFF) + endif () + # Otherwise `make install` puts googletest into SZ3's prefix. + set(INSTALL_GTEST OFF) + FetchContent_MakeAvailable(googletest) +else() + message(STATUS "Using locally installed GTest") +endif() + +include(CTest) +include(GoogleTest) + +# Collect all test source files +file(GLOB TEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/modules/*.cpp") + +foreach (TEST_SRC IN LISTS TEST_SOURCES) + get_filename_component(TEST_NAME ${TEST_SRC} NAME_WE) + message(STATUS "Test target = ${TEST_NAME}") + add_executable(${TEST_NAME} ${TEST_SRC}) + target_link_libraries(${TEST_NAME} PRIVATE GTest::gtest_main SZ3) + # Automatically discover tests for this executable + # Use PRE_TEST mode to avoid running executable at build time (important for Windows DLL path issues) + gtest_discover_tests(${TEST_NAME} DISCOVERY_MODE PRE_TEST) +endforeach() +target_compile_definitions(test_previous_data_version PRIVATE SZ3_TEST_STREAMS="${CMAKE_CURRENT_SOURCE_DIR}/streams") + +# GCC refuses -ffast-math but not these options, which together on an FMA target let it drop floating-point NaN tests. +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + add_executable(biomd_finite_math biomd_finite_math.cpp) + target_link_libraries(biomd_finite_math PRIVATE SZ3core) + target_compile_options(biomd_finite_math PRIVATE -funsafe-math-optimizations -ffinite-math-only -march=x86-64-v3) + # it runs only where the machine has the instructions -march=x86-64-v3 uses + include(CheckCXXSourceRuns) + set(CMAKE_REQUIRED_FLAGS -march=x86-64-v3) + check_cxx_source_runs("int main() { return __builtin_cpu_supports(\"avx2\") && __builtin_cpu_supports(\"fma\") ? 0 : 1; }" + SZ3_HOST_HAS_X86_64_V3) + unset(CMAKE_REQUIRED_FLAGS) + if (SZ3_HOST_HAS_X86_64_V3) + add_test(NAME SZ3_BioMD_NaNUnderFiniteMath COMMAND biomd_finite_math) + endif () +endif () + +# A program built without OpenMP reads back what one built with it wrote with conf.openmp = true. +if (OpenMP_FOUND) + add_executable(openmp_stream_with openmp_stream.cpp) + target_link_libraries(openmp_stream_with PRIVATE SZ3) + add_executable(openmp_stream_without openmp_stream.cpp) + target_link_libraries(openmp_stream_without PRIVATE SZ3core) + + add_test(NAME SZ3_OpenMPStream_WriteWithOpenMP + COMMAND openmp_stream_with with write ${CMAKE_CURRENT_BINARY_DIR}/stream_with) + # More than one thread, or the stream holds one chunk and the read below proves nothing. + set_tests_properties(SZ3_OpenMPStream_WriteWithOpenMP PROPERTIES + ENVIRONMENT OMP_NUM_THREADS=4 FIXTURES_SETUP stream_with) + add_test(NAME SZ3_OpenMPStream_ReadWithoutOpenMP + COMMAND openmp_stream_without without read ${CMAKE_CURRENT_BINARY_DIR}/stream_with) + set_tests_properties(SZ3_OpenMPStream_ReadWithoutOpenMP PROPERTIES FIXTURES_REQUIRED stream_with) + # ALGO_BIOMD coordinates, split along the atoms + add_test(NAME SZ3_OpenMPStream_BioMDWriteWithOpenMP + COMMAND openmp_stream_with with write ${CMAKE_CURRENT_BINARY_DIR}/stream_biomd biomd) + set_tests_properties(SZ3_OpenMPStream_BioMDWriteWithOpenMP PROPERTIES + ENVIRONMENT OMP_NUM_THREADS=4 FIXTURES_SETUP stream_biomd) + add_test(NAME SZ3_OpenMPStream_BioMDReadWithoutOpenMP + COMMAND openmp_stream_without without read ${CMAKE_CURRENT_BINARY_DIR}/stream_biomd) + set_tests_properties(SZ3_OpenMPStream_BioMDReadWithoutOpenMP PROPERTIES FIXTURES_REQUIRED stream_biomd) + + add_executable(openmp_regions openmp_regions.cpp) + target_link_libraries(openmp_regions PRIVATE SZ3) + add_test(NAME SZ3_OpenMP_ChunkThrows COMMAND openmp_regions throw) + set_tests_properties(SZ3_OpenMP_ChunkThrows PROPERTIES TIMEOUT 60) + # MinGW's libgomp crashes in nested parallel regions (GCC 15.2 and 16.2). + if (NOT MINGW) + add_test(NAME SZ3_OpenMP_FewerThreadsThanAsked COMMAND openmp_regions nested) + set_tests_properties(SZ3_OpenMP_FewerThreadsThanAsked PROPERTIES + ENVIRONMENT "OMP_NUM_THREADS=4,4;OMP_THREAD_LIMIT=8;OMP_MAX_ACTIVE_LEVELS=2" TIMEOUT 300) + endif () +endif () diff --git a/tools/test/biomd_finite_math.cpp b/tools/test/biomd_finite_math.cpp new file mode 100644 index 00000000..731f2280 --- /dev/null +++ b/tools/test/biomd_finite_math.cpp @@ -0,0 +1,31 @@ +// ALGO_BIOMD built with GCC's -funsafe-math-optimizations and -ffinite-math-only on an FMA target (each allowed on +// its own, unlike -ffast-math): a NaN in a frame that is not trailing fill must come back as NaN, the chunk stored +// losslessly. Exits 0 when it does. + +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" + +int main() { + const size_t frames = 2, atoms = 300, at = 3 * 17 + 1; + std::vector x(frames * atoms * 3); + for (size_t i = 0; i < x.size(); i++) x[i] = 1.0f + 0.01f * float(i % 900) + 0.001f * float(i / 900); + const unsigned nan_bits = 0x7fc00000u; // set as bits, so the compiler cannot fold it away + std::memcpy(&x[at], &nan_bits, sizeof(float)); + SZ3::Config conf(frames, atoms, 3); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 5e-4; + size_t size = 0; + std::unique_ptr cmp(SZ_compress(conf, x.data(), size)); + SZ3::Config dec; + std::unique_ptr out(SZ_decompress(dec, cmp.get(), size)); + unsigned bits; + std::memcpy(&bits, &out[at], sizeof(float)); + const bool kept = std::memcmp(out.get(), x.data(), x.size() * sizeof(float)) == 0; + std::printf("algorithm %d, the NaN came back as bits %08x: %s\n", int(dec.cmprAlgo), bits, kept ? "kept" : "LOST"); + return kept ? 0 : 1; +} diff --git a/tools/test/bundledZstdIsolation.sh b/tools/test/bundledZstdIsolation.sh new file mode 100755 index 00000000..0f12a598 --- /dev/null +++ b/tools/test/bundledZstdIsolation.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# What a build with SZ3_USE_BUNDLED_ZSTD=ON is allowed to hand the rest of the machine: the +# vendored Zstd has to reach SZ3's consumers as a linkable target and as nothing else -- no header +# in the prefix, none reachable through the export, no symbol in a shared library. +# +# tools/test/bundledZstdIsolation.sh +# +# Assert which Zstd was used and what escaped, never just that the build exited 0. No set -e: every +# check below is explicit, and the total at the bottom catches a section that stopped early. +set -u +BUILD=$(cd "$1" && pwd) +PREFIX=$(cd "$2" && pwd) +LIBDIR=$PREFIX/lib +[ -d "$LIBDIR" ] || LIBDIR=$PREFIX/lib64 +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +cd "$WORK" || exit 1 + +pass=0; fail=0; skip=0 +ok() { echo "PASS $1"; pass=$((pass+1)); } +bad() { echo "FAIL $1"; shift; for l in "$@"; do echo " $l"; done; fail=$((fail+1)); } +skipped() { echo "SKIP $1"; skip=$((skip+1)); } + +SZ3BIN=$BUILD/tools/sz3/sz3 +[ -x "$SZ3BIN" ] || SZ3BIN=$BUILD/tools/sz3/sz3.exe +# Required, not skipped: a missing binary must never look like a passing check. +if [ ! -x "$SZ3BIN" ]; then + echo "no sz3 binary under $BUILD/tools/sz3, so there is nothing to ask which Zstd it linked" + exit 1 +fi +# The one place a Mach-O build differs; everything else here reads files. +linked_libs() { + if command -v ldd > /dev/null 2>&1; then ldd "$1" 2>&1 + elif command -v otool > /dev/null 2>&1; then otool -L "$1" 2>&1 + else echo ""; fi +} + +echo "=== bundled Zstd: build $BUILD, prefix $PREFIX ===" + +# ---------------------------------------------------------------- 1. it really is the bundled one +if [ -e "$LIBDIR/libsz3_zstd.a" ] || [ -e "$LIBDIR/sz3_zstd.lib" ]; then + ok "bundled-zstd-archive-installed" +else + bad "bundled-zstd-archive-installed" "no libsz3_zstd.a under $LIBDIR" +fi +linked_libs "$SZ3BIN" > linked.log +if grep -q 'libzstd\.so\|libzstd\.[0-9]*\.dylib\|libzstd\.dylib' linked.log; then + bad "no-system-libzstd-linked" "asked for the bundled Zstd and linked the system one anyway" \ + "$(grep 'libzstd' linked.log)" +else + ok "no-system-libzstd-linked" +fi + +# ---------------------------------------------------------------- 2. what the export offers +TARGETS=$LIBDIR/cmake/SZ3/SZ3Targets.cmake +if [ -f "$TARGETS" ] && grep -q 'SZ3::zstd' "$TARGETS"; then + ok "export-names-sz3-zstd" +else + bad "export-names-sz3-zstd" "$TARGETS does not name SZ3::zstd, so a consumer has nothing to link" +fi +# An absolute path into this machine's build tree pins the export to the machine that produced it. +grep -h 'libsz3_zstd' "$LIBDIR"/cmake/SZ3/SZ3Targets*.cmake 2>/dev/null > vendored.log +if [ ! -s vendored.log ]; then + # Not a pass: SZ3::zstd is an archive the export has to locate by path, so naming it nowhere + # means a consumer resolves the name itself and this check had nothing to look at. + bad "export-references-the-vendored-zstd-relocatably" \ + "no SZ3Targets*.cmake under $LIBDIR/cmake/SZ3 names libsz3_zstd at all" +elif grep -qv '_IMPORT_PREFIX' vendored.log; then + bad "export-references-the-vendored-zstd-relocatably" \ + "referenced by an absolute path, so the export cannot move" "$(cat vendored.log)" +else + ok "export-references-the-vendored-zstd-relocatably" +fi + +# ---------------------------------------------------------------- 3. nothing leaked into the prefix +# These are the names the system Zstd's own package owns. Installing any of them makes SZ3 +# uninstallable beside it, which is what Debian and Fedora reject a package for. +leaked= +for f in include/zstd.h include/zdict.h include/zbuff.h include/zstd_errors.h \ + lib/libzstd.a lib/libzstd.so include/SZ3/bundled_zstd/zstd.h; do + [ -e "$PREFIX/$f" ] && leaked="$leaked $f" +done +if [ -n "$leaked" ]; then + bad "no-zstd-file-the-system-package-owns" "installed:$leaked" +else + ok "no-zstd-file-the-system-package-owns" +fi +find "$PREFIX" -name 'zstd.h' > anyheader.log 2>/dev/null +if [ -s anyheader.log ]; then + bad "no-zstd-h-anywhere-in-the-prefix" "$(cat anyheader.log)" +else + ok "no-zstd-h-anywhere-in-the-prefix" +fi + +# A consumer compiles with every exported include directory on its command line, so a zstd.h +# sitting in one of them shadows the system Zstd's header for the whole translation unit. +grep -ho 'INTERFACE_INCLUDE_DIRECTORIES "[^"]*"' "$LIBDIR"/cmake/SZ3/*.cmake 2>/dev/null \ + | sed 's/.*"\(.*\)"/\1/' | tr ';' '\n' | sed "s|\${_IMPORT_PREFIX}|$PREFIX|" \ + | sed '/^$/d' | sort -u > incdirs.log +# Counted first: with no include directories to walk, the check below has nothing to look inside +# and would pass on an export that declares none. +if [ -s incdirs.log ]; then + ok "export-declares-include-directories" + sed 's/^/ /' incdirs.log +else + bad "export-declares-include-directories" "no INTERFACE_INCLUDE_DIRECTORIES under $LIBDIR/cmake/SZ3" +fi +reachable= +while read -r d; do + [ -e "$d/zstd.h" ] && reachable="$reachable $d" +done < incdirs.log +if [ -n "$reachable" ]; then + bad "no-zstd-h-reachable-from-an-exported-include-dir" "a zstd.h in:$reachable" +else + ok "no-zstd-h-reachable-from-an-exported-include-dir" +fi + +# ---------------------------------------------------------------- 4. nothing leaked into the ABI +# A shared library that re-exports ZSTD_* interposes on every other Zstd in the process. +find "$LIBDIR" -maxdepth 1 \( -name '*.so' -o -name '*.so.*' -o -name '*.dylib' \) > shared.log 2>/dev/null +if ! command -v nm > /dev/null 2>&1; then + skipped "no-shared-library-exports-zstd-symbols (no nm)" +elif [ ! -s shared.log ]; then + # A named skip, not a pass: BUILD_SHARED_LIBS=OFF installs archives, and an archive carries the + # symbols without exporting them. + skipped "no-shared-library-exports-zstd-symbols ($LIBDIR holds no shared library)" +else + sed 's/^/ /' shared.log + exporters=; readable=0 + while read -r lib; do + nm -D --defined-only "$lib" > syms.log 2>/dev/null + # Counted, because nm reporting nothing for every library reads exactly like a clean result. + [ -s syms.log ] && readable=$((readable+1)) + grep -q ' T \(ZSTD_\|ZDICT_\|ZBUFF_\)' syms.log && exporters="$exporters $lib" + done < shared.log + if [ -n "$exporters" ]; then + bad "no-shared-library-exports-zstd-symbols" "exports Zstd's symbols:$exporters" + elif [ "$readable" = 0 ]; then + bad "no-shared-library-exports-zstd-symbols" \ + "nm read a dynamic symbol table out of none of them, so nothing was inspected" + else + ok "no-shared-library-exports-zstd-symbols" + fi +fi + +echo +echo " $pass passed, $fail failed, $skip skipped" + +# Raise this with the check it comes with. A section that stops early otherwise shows only as a +# smaller number at the bottom that nobody compares. +EXPECTED=9 +ran=$((pass + fail + skip)) +if [ "$ran" -ne "$EXPECTED" ]; then + echo " the suite accounted for $ran checks, not $EXPECTED" + exit 1 +fi +exit $((fail > 0)) diff --git a/tools/test/checkMaxError.sh b/tools/test/checkMaxError.sh new file mode 100755 index 00000000..d1d8fc3b --- /dev/null +++ b/tools/test/checkMaxError.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# The error sz3 -a reports, against the bound it was asked for. +# +# tools/test/checkMaxError.sh +# +# The parse is its own assertion: sed exits 0 when it matches nothing, so a renamed or missing line +# leaves an empty string that compares below the bound and reads as a compressor that stayed in it. +set -u +LOG=$1 +TOL=$2 + +if [ ! -f "$LOG" ]; then + echo "FAIL max-absolute-error-was-reported" + echo " no such file: $LOG" + exit 1 +fi +if [ -z "$TOL" ]; then + echo "FAIL max-absolute-error-within-tolerance" + echo " no tolerance given, so there is nothing to compare against" + exit 1 +fi + +ERR=$(grep -E 'Max absolute error =' "$LOG" \ + | sed -E 's/.*Max absolute error = *([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?).*/\1/') +if [ -z "$ERR" ]; then + echo "FAIL max-absolute-error-was-reported" + echo " no 'Max absolute error =' line in $LOG" + echo " last lines:" + tail -10 "$LOG" | sed 's/^/ /' + exit 1 +fi +echo "PASS max-absolute-error-was-reported" + +if awk -v err="$ERR" -v tol="$TOL" 'BEGIN { exit (err > tol) }'; then + echo "PASS max-absolute-error-within-tolerance" + echo " $ERR <= $TOL" +else + echo "FAIL max-absolute-error-within-tolerance" + echo " $ERR exceeds the requested bound $TOL" + exit 1 +fi diff --git a/tools/test/check_workflow_env.py b/tools/test/check_workflow_env.py new file mode 100755 index 00000000..1f0ffadd --- /dev/null +++ b/tools/test/check_workflow_env.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Every env key a workflow interpolates has to be declared in a scope that reaches it. + +GitHub interpolates an undeclared key to the empty string, so a step keeps running with an +argument missing. That is how splitting one workflow into three left `sz3 -3 -M ` behind: the +compressor printed its usage text and exited non-zero, which reads as a compression failure. + +Scope counts for as much as spelling. A key declared in one job does not exist in another, a +step's own env: reaches that step alone, and a $GITHUB_ENV write reaches only the later steps of +the same job. Checking every reference against the union of every env: block in the file would +pass the one workflow this script exists to catch, so each scope is tracked separately. + + python3 tools/test/check_workflow_env.py [--workflow-dir .github/workflows] + +Reads the workflow files as text rather than YAML so it needs nothing installed. +""" + +import argparse +import pathlib +import re +import sys + +# `${{ env.NAME }}`, with the whitespace GitHub allows on either side of the name. +REFERENCE = re.compile(r"\$\{\{\s*env\.([A-Za-z_][A-Za-z0-9_-]*)\s*\}\}") +# A key inside an `env:` mapping: `NAME: value`, where value may be empty or continue on later +# lines. An empty value is a finding too -- it reaches a program exactly as an undeclared key does. +ENV_KEY = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_-]*):(.*)$") +ENV_BLOCK = re.compile(r"^(\s*)env:\s*(#.*)?$") +# `echo "NAME=..." >> $GITHUB_ENV` and the PowerShell `"NAME=..." | Out-File ... GITHUB_ENV`, which +# declare a key for later steps without an env: block. +RUNTIME_KEY = re.compile(r"[\"']?([A-Za-z_][A-Za-z0-9_-]*)=") +MAPPING_KEY = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_-]*):") +STEPS_BLOCK = re.compile(r"^(\s*)steps:\s*(#.*)?$") +LIST_ITEM = re.compile(r"^(\s*)-(\s|$)") + +EMPTY_VALUES = ("", "''", '""') + + +def line_scopes(lines): + """For each line, the (job, step) it sits in -- the two scopes narrower than the workflow. + + `job` is the job id, or None for a line outside `jobs:`. `step` is the step's index within + that job, or None for a line that belongs to the job itself (runs-on:, strategy:, ...). + """ + scopes = [] + in_jobs = False + job = job_indent = None + step = step_indent = steps_indent = None + for line in lines: + stripped = line.rstrip("\n") + body = stripped.strip() + if body and not body.startswith("#"): + indent = len(stripped) - len(stripped.lstrip()) + if indent == 0 and MAPPING_KEY.match(stripped): + # A new top-level key closes every job and step under the previous one. + in_jobs = body.split(":", 1)[0] == "jobs" + job = job_indent = None + step = step_indent = steps_indent = None + elif in_jobs and job_indent is None: + # The first key under `jobs:` fixes the indent the job ids sit at. + match = MAPPING_KEY.match(stripped) + if match: + job_indent = indent + job = match.group(2) + elif in_jobs and indent == job_indent and MAPPING_KEY.match(stripped): + job = MAPPING_KEY.match(stripped).group(2) + step = step_indent = steps_indent = None + elif in_jobs and job is not None: + item = LIST_ITEM.match(stripped) + if steps_indent is not None and indent <= steps_indent and not ( + item and indent == step_indent + ): + # A key at or above `steps:` is the job speaking again, not another step. + step = step_indent = steps_indent = None + if steps_indent is None: + match = STEPS_BLOCK.match(stripped) + if match: + steps_indent = len(match.group(1)) + elif item and (step_indent is None or indent == step_indent): + step_indent = indent + step = 0 if step is None else step + 1 + scopes.append((job, step)) + return scopes + + +def declarations(lines): + """Each key an env: mapping declares, as (index of the env: line, name, value).""" + block_indent = key_indent = block_line = None + for index, line in enumerate(lines): + stripped = line.rstrip("\n") + if not stripped.strip() or stripped.lstrip().startswith("#"): + continue + indent = len(stripped) - len(stripped.lstrip()) + if block_indent is not None and indent <= block_indent: + block_indent = key_indent = None + if block_indent is not None: + match = ENV_KEY.match(stripped) + if match and (key_indent is None or len(match.group(1)) == key_indent): + key_indent = len(match.group(1)) + yield block_line, match.group(2), match.group(3).strip() + continue + block = ENV_BLOCK.match(stripped) + if block: + block_indent = len(block.group(1)) + key_indent = None + block_line = index + + +def runtime_declarations(lines): + """Names written to $GITHUB_ENV, as (line index, name). Only later steps of the job see them.""" + for index, line in enumerate(lines): + if "GITHUB_ENV" not in line: + continue + for match in RUNTIME_KEY.finditer(line): + if match.group(1) != "GITHUB_ENV": + yield index, match.group(1) + + +def check(path): + text = path.read_text() + lines = text.splitlines() + scopes = line_scopes(lines) + + workflow = {} + jobs = {} # job -> {name: value} + steps = {} # (job, step) -> {name: value} + for index, name, value in declarations(lines): + job, step = scopes[index] + if job is None: + target = workflow + elif step is None: + target = jobs.setdefault(job, {}) + else: + target = steps.setdefault((job, step), {}) + target[name] = value + + # job -> name -> the earliest step that writes it; a write is visible from the next step on. + runtime = {} + for index, name in runtime_declarations(lines): + job, step = scopes[index] + if job is None or step is None: + continue + written = runtime.setdefault(job, {}) + written[name] = min(step, written.get(name, step)) + + problems = [] + for match in REFERENCE.finditer(text): + name = match.group(1) + index = text.count("\n", 0, match.start()) + job, step = scopes[index] + # GitHub's resolution order, innermost first. Anything outside this chain is a different + # scope. + chain = [] + if step is not None: + chain.append(steps.get((job, step), {})) + if runtime.get(job, {}).get(name, step) < step: + chain.append({name: "<$GITHUB_ENV>"}) + if job is not None: + chain.append(jobs.get(job, {})) + chain.append(workflow) + + value = next((scope[name] for scope in chain if name in scope), None) + if value is None: + problems.append(f"{path}:{index + 1}: env.{name} is not declared in any scope that reaches it") + elif value in EMPTY_VALUES: + problems.append(f"{path}:{index + 1}: env.{name} is declared with an empty value") + return problems + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workflow-dir", default=".github/workflows", type=pathlib.Path) + args = parser.parse_args() + + paths = sorted(args.workflow_dir.glob("*.yml")) + sorted(args.workflow_dir.glob("*.yaml")) + if not paths: + print(f"no workflows under {args.workflow_dir}", file=sys.stderr) + return 1 + + problems = [] + for path in paths: + problems += check(path) + for problem in problems: + print(problem, file=sys.stderr) + print(f"checked {len(paths)} workflows, {len(problems)} undeclared or empty env references") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test/consumeInstalledSZ3.sh b/tools/test/consumeInstalledSZ3.sh new file mode 100755 index 00000000..1e950e6a --- /dev/null +++ b/tools/test/consumeInstalledSZ3.sh @@ -0,0 +1,412 @@ +#!/bin/bash +# Every shape a downstream project takes when it consumes an installed SZ3, asserted against an +# install tree. +# +# tools/test/consumeInstalledSZ3.sh [] +# +# The second argument is appended to CMAKE_PREFIX_PATH, for an HDF5 that is not on the default +# search path. CMAKE_PREFIX_PATH from the environment is passed through as well. +# +# Assert which shape built and what the export said, never just that cmake exited 0. No set -e: +# every check below is explicit, and the total at the bottom catches a section that stopped early. +set -u +PREFIX=$(cd "$1" && pwd) +EXTRA_PREFIX=${2:-} +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +cd "$WORK" || exit 1 + +# Under MSYS2 and Cygwin, cmake is a native Windows program while this shell deals in /d/... paths +# it cannot open. A prefix handed over unconverted is simply not searched, and cmake then finds SZ3 +# wherever else it can. +native() { if command -v cygpath > /dev/null 2>&1; then cygpath -m "$1"; else echo "$1"; fi; } +LIBDIR=$PREFIX/lib +[ -d "$LIBDIR" ] || LIBDIR=$PREFIX/lib64 +# SZ3Config.cmake decides whether SZ3::hdf5sz3 exists by looking for exactly this file, and +# BUILD_H5Z_FILTER is off by default -- so a prefix without it is sound, not broken. +HAVE_FILTER=0 +[ -f "$LIBDIR/cmake/SZ3/HDF5SZ3.cmake" ] && HAVE_FILTER=1 +# ... but only when nothing else in the prefix says the filter was built. tools/H5Z-SZ3 installs its +# headers and its binary through install() rules separate from install(EXPORT), so either one +# arriving without HDF5SZ3.cmake means the export was dropped rather than never built. +# Matched on *hdf5sz3*, not lib*: MSVC names them hdf5sz3.dll and hdf5sz3.lib, with no lib prefix. +if [ "$HAVE_FILTER" = 0 ]; then + traces=$(ls -d "$PREFIX"/include/hdf5_sz3 2>/dev/null + ls "$PREFIX"/lib*/*hdf5sz3* "$PREFIX"/lib*/plugin/*hdf5sz3* "$PREFIX"/bin/*hdf5sz3* 2>/dev/null) + if [ -n "$traces" ]; then + echo "FAIL the install carries the filter but the export does not declare it" + echo " missing: $LIBDIR/cmake/SZ3/HDF5SZ3.cmake" + echo " present:" + echo "$traces" | sed 's/^/ /' + echo " SZ3::hdf5sz3 does not exist for any consumer, and every filter check below" + echo " would have been skipped as though this SZ3 had been built without the filter." + exit 1 + fi +fi +CMPFX=$(native "$PREFIX") +[ -n "$EXTRA_PREFIX" ] && CMPFX="$CMPFX;$(native "$(cd "$EXTRA_PREFIX" && pwd)")" +[ -n "${CMAKE_PREFIX_PATH:-}" ] && CMPFX="$CMPFX;$CMAKE_PREFIX_PATH" +# Windows resolves a DLL through PATH and has no RPATH for an install tree to be recorded in, so +# /bin -- where the install rule puts the runtime artifact -- is the only way the consumer +# built below reaches hdf5sz3. POSIX form: bash splits PATH on ':', and the MSYS2 runtime converts +# the whole variable when it spawns a native program. +export PATH="$PREFIX/bin:$PATH" + +pass=0; fail=0; skip=0 +ok() { echo "PASS $1"; pass=$((pass+1)); } +bad() { echo "FAIL $1"; shift; for l in "$@"; do echo " $l"; done; fail=$((fail+1)); } +skipped() { echo "SKIP $1"; skip=$((skip+1)); } +# reason , then the names it covers +skip_all() { reason=$1; shift; for n in "$@"; do skipped "$n ($reason)"; done; } +# want +want() { if grep -qF "$2" "$3"; then ok "$1"; else bad "$1" "expected to find: $2" "got:" "$(head -10 "$3")"; fi; } + +echo "=== prefix $PREFIX ===" +echo " cmake prefix path $CMPFX" + +# ---------------------------------------------------------------- 1. C++, unversioned targets +# target_link_libraries(app PRIVATE SZ3 hdf5sz3): the spelling with no SZ3:: namespace, which CMake +# turns into a bare -lhdf5sz3 unless SZ3Config.cmake defines the alias. +if [ "$HAVE_FILTER" = 1 ]; then +mkdir -p cxx +cat > cxx/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(consumer CXX) +find_package(SZ3 REQUIRED) +add_executable(app main.cpp) +target_link_libraries(app PRIVATE SZ3 hdf5sz3) +EOF +cat > cxx/main.cpp <<'EOF' +#include +#include +#include +int main() { + printf("%s\n", H5Zregister(H5PLget_plugin_info()) < 0 ? "INIT FAILED" : "INIT OK"); + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + SZ3::Config conf(100); + conf.absErrorBound = 1e-3; + set_SZ3_conf_to_H5(dcpl, conf); + printf("NFILTERS %d\n", (int)H5Pget_nfilters(dcpl)); + return 0; +} +EOF +# A multi-config generator defaults an unqualified build to Debug and writes it to b/, so +# the checks below look for a binary that is not there. Naming the config and its output directory +# puts it at b/ either way: with no CMAKE_BUILD_TYPE here, a single-config generator ignores both. +if cmake -S cxx -B cxx/b -DCMAKE_PREFIX_PATH="$CMPFX" \ + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE="$(native "$WORK")/cxx/b" > cxx/cfg.log 2>&1; then + ok "cxx-consumer-configures" + if cmake --build cxx/b --config Release --parallel 4 > cxx/build.log 2>&1; then + ok "cxx-consumer-builds" + if ./cxx/b/app > cxx/run.log 2>&1; then + ok "cxx-consumer-runs" + else + bad "cxx-consumer-runs" "exit $?" "$(tail -5 cxx/run.log)" + fi + else + bad "cxx-consumer-builds" "$(tail -15 cxx/build.log)" + bad "cxx-consumer-runs" "nothing was built to run" + : > cxx/run.log + fi +else + bad "cxx-consumer-configures" "$(tail -15 cxx/cfg.log)" + bad "cxx-consumer-builds" "nothing was configured to build" + bad "cxx-consumer-runs" "nothing was built to run" + : > cxx/run.log +fi +want "cxx-consumer-initializes-the-filter" "INIT OK" cxx/run.log +want "cxx-consumer-sets-one-filter-on-a-dcpl" "NFILTERS 1" cxx/run.log +else +skip_all "this SZ3 was built without BUILD_H5Z_FILTER, so the export carries no hdf5sz3" \ + cxx-consumer-configures cxx-consumer-builds cxx-consumer-runs \ + cxx-consumer-initializes-the-filter cxx-consumer-sets-one-filter-on-a-dcpl +fi + +# ---------------------------------------------------------------- 2. C only, namespaced target +# A C project never enables CXX, so anything SZ3Config.cmake resolves per-language has to cope. +# REQUIRED, and no if (SZ3_FOUND) around the executable: written the other way this builds nothing +# whenever SZ3 is not found, which is exactly when a C-only consumer is broken. +if [ "$HAVE_FILTER" = 1 ]; then +mkdir -p conly +cat > conly/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(conly C) +find_package(SZ3 REQUIRED) +add_executable(c1 main.c) +target_link_libraries(c1 PRIVATE SZ3::hdf5sz3) +EOF +printf 'int main(void){return 0;}\n' > conly/main.c +if cmake -S conly -B conly/b -DCMAKE_PREFIX_PATH="$CMPFX" \ + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE="$(native "$WORK")/conly/b" > conly/cfg.log 2>&1; then + ok "c-only-consumer-configures" + if cmake --build conly/b --config Release --parallel 4 > conly/build.log 2>&1; then + ok "c-only-consumer-builds" + else + bad "c-only-consumer-builds" "$(tail -15 conly/build.log)" + fi +else + bad "c-only-consumer-configures" "$(tail -15 conly/cfg.log)" + bad "c-only-consumer-builds" "nothing was configured to build" +fi +if [ -x conly/b/c1 ] || [ -x conly/b/c1.exe ]; then + ok "c-only-consumer-produces-an-executable" +else + bad "c-only-consumer-produces-an-executable" "no c1 under conly/b" +fi +else +skip_all "this SZ3 was built without BUILD_H5Z_FILTER, so the export carries no hdf5sz3" \ + c-only-consumer-configures c-only-consumer-builds c-only-consumer-produces-an-executable +fi + +# ---------------------------------------------------------------- 2b. C only, SZ3::SZ3c +# libSZ3c is installed with the tools; a C project finds it through the export and gets NULL, not an +# exit, for what it does not support. +if [ -f "$LIBDIR/cmake/SZ3/SZ3c.cmake" ]; then +mkdir -p sz3c +cat > sz3c/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +# CXX only so the linker brings the C++ runtime a static libSZ3c needs; the program is C. +project(sz3c_consumer C CXX) +find_package(SZ3 REQUIRED COMPONENTS SZ3c) +add_executable(c2 main.c) +target_link_libraries(c2 PRIVATE SZ3::SZ3c) +EOF +cat > sz3c/main.c <<'EOF' +#include +int main(void) { + float d[1000]; + for (int i = 0; i < 1000; i++) d[i] = (float)(i % 37) * 0.1f; /* no libm to link */ + size_t n = 0; + unsigned char *c = SZ_compress_args(SZ_FLOAT, d, &n, ABS, 1e-3, 0, 0, 0, 0, 0, 0, 1000); + if (!c) return 1; + float *o = (float *)SZ_decompress(SZ_FLOAT, c, n, 0, 0, 0, 0, 1000); + if (!o) return 2; + for (int i = 0; i < 1000; i++) + if (!(o[i] - d[i] <= 1e-3f && d[i] - o[i] <= 1e-3f)) return 3; + if (SZ_decompress(SZ_FLOAT, c, n, 0, 0, 0, 0, 999)) return 4; /* wrong dimensions */ + if (SZ_compress_args(SZ_INT8, d, &n, ABS, 1e-3, 0, 0, 0, 0, 0, 0, 1000)) return 5; + if (SZ_compress_args(SZ_FLOAT, d, &n, PSNR, 1e-3, 0, 0, 0, 0, 0, 0, 1000)) return 6; + free_buf(o); + free_buf(c); + return 0; +} +EOF +if cmake -S sz3c -B sz3c/b -DCMAKE_PREFIX_PATH="$CMPFX" \ + -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE="$(native "$WORK")/sz3c/b" > sz3c/cfg.log 2>&1 && + cmake --build sz3c/b --config Release --parallel 4 > sz3c/build.log 2>&1; then + ok "sz3c-consumer-builds" + if sz3c/b/c2 2> sz3c/run.log; then + ok "sz3c-consumer-runs" + else + bad "sz3c-consumer-runs" "exit $?" "$(tail -5 sz3c/run.log)" + fi +else + bad "sz3c-consumer-builds" "$(tail -15 sz3c/cfg.log sz3c/build.log 2>/dev/null)" + bad "sz3c-consumer-runs" "nothing was built" +fi +else +skip_all "this SZ3 was installed without its tools, so the export carries no SZ3c" \ + sz3c-consumer-builds sz3c-consumer-runs +fi + +# ---------------------------------------------------------------- 3. the exported include paths +# A build directory or a third party's include tree named here reaches every consumer, and points +# at something their machine need not have. +grep -ho 'INTERFACE_INCLUDE_DIRECTORIES "[^"]*"' "$LIBDIR"/cmake/SZ3/*.cmake 2>/dev/null \ + | sed 's/.*"\(.*\)"/\1/' | tr ';' '\n' | sed '/^$/d' | sort -u > inc.log +# Counted first: with no include directories to look at, the check below has nothing to reject and +# would pass on an export that declares none. +if [ -s inc.log ]; then + ok "export-declares-include-directories" + sed 's/^/ /' inc.log +else + bad "export-declares-include-directories" "no INTERFACE_INCLUDE_DIRECTORIES under $LIBDIR/cmake/SZ3" +fi +# grep -v, not a !-inverted grep: this has to report the offending line. +if grep -v '^\${_IMPORT_PREFIX}/' inc.log > outside.log; then + bad "exported-include-dirs-stay-in-prefix" "outside the install prefix:" "$(cat outside.log)" +else + ok "exported-include-dirs-stay-in-prefix" +fi + +# ---------------------------------------------------------------- 4. no HDF5 on the machine +# SZ3Config.cmake reports not-found and returns, rather than ending the consumer's configure or +# handing them a link line they cannot use. GROMACS's GMX_USE_SZ3=AUTO depends on this shape. +# Only the filter makes SZ3 depend on HDF5 at all, so without it there is no declining to observe. +if [ "$HAVE_FILTER" = 1 ]; then +mkdir -p probe +cat > probe/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(probe CXX) +find_package(SZ3 QUIET) +if (SZ3_FOUND) + message(FATAL_ERROR "SZ3 reported itself found with no HDF5 to be had") +endif () +# GROMACS builds its own copy of SZ3 next, which fails on any SZ3 target left behind. +if (TARGET SZ3::SZ3 OR TARGET SZ3::SZ3core OR TARGET SZ3::hdf5sz3) + message(FATAL_ERROR "SZ3 declined but left its targets behind") +endif () +EOF +if cmake -S probe -B probe/b -DCMAKE_PREFIX_PATH="$CMPFX" \ + -DCMAKE_DISABLE_FIND_PACKAGE_HDF5=ON > probe/cfg.log 2>&1; then + ok "sz3-declines-when-hdf5-cannot-be-found" +else + bad "sz3-declines-when-hdf5-cannot-be-found" "$(tail -15 probe/cfg.log)" +fi +else +skip_all "this SZ3 was built without BUILD_H5Z_FILTER, so it depends on no HDF5" \ + sz3-declines-when-hdf5-cannot-be-found +fi + +# ---------------------------------------------------------------- 5. no Zstd on the machine +# The other way SZ3Config.cmake declines. It resolves a system Zstd with find_library, which no +# CMAKE_DISABLE_FIND_PACKAGE_ reaches, so point the library search at an empty root instead. +# A bundled-Zstd build carries SZ3::zstd in the export and never runs that branch. +if grep -q "TARGET SZ3::zstd" "$LIBDIR/cmake/SZ3/SZ3Targets.cmake" 2>/dev/null || + [ -n "$(find "$LIBDIR" -name '*sz3_zstd*' 2>/dev/null)" ]; then + skip_all "this SZ3 bundles its own Zstd, so it looks for none here" \ + sz3-declines-when-zstd-cannot-be-found +else + mkdir -p zprobe/empty + cp probe/CMakeLists.txt zprobe/CMakeLists.txt 2>/dev/null || cat > zprobe/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(probe CXX) +find_package(SZ3 QUIET) +if (SZ3_FOUND) + message(FATAL_ERROR "SZ3 reported itself found with no Zstd to be had") +endif () +# GROMACS builds its own copy of SZ3 next, which fails on any SZ3 target left behind. +if (TARGET SZ3::SZ3 OR TARGET SZ3::SZ3core OR TARGET SZ3::hdf5sz3) + message(FATAL_ERROR "SZ3 declined but left its targets behind") +endif () +EOF + if cmake -S zprobe -B zprobe/b -DCMAKE_PREFIX_PATH="$CMPFX" \ + -DCMAKE_FIND_ROOT_PATH="$(native "$WORK")/zprobe/empty" \ + -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY > zprobe/cfg.log 2>&1; then + ok "sz3-declines-when-zstd-cannot-be-found" + else + bad "sz3-declines-when-zstd-cannot-be-found" "$(tail -15 zprobe/cfg.log)" + fi +fi + +# ---------------------------------------------------------------- 6. no SZ3 internals exported +# Another SZ3 copy in the same process would take over any SZ3 function these libraries export. +for lib in hdf5sz3 SZ3c; do + if [ -f "$LIBDIR/lib$lib.so" ] && command -v nm > /dev/null 2>&1; then + if ! nm -DC --defined-only "$LIBDIR/lib$lib.so" > "nm_$lib.log" 2>&1; then + bad "$lib-exports-no-sz3-functions" "nm failed:" "$(head -3 "nm_$lib.log")" + elif grep -E "^[0-9a-f]+ [TWVu] SZ3::" "nm_$lib.log" > "exports_$lib.log"; then + bad "$lib-exports-no-sz3-functions" "$(wc -l < "exports_$lib.log") exported, for example:" \ + "$(head -3 "exports_$lib.log")" + else + ok "$lib-exports-no-sz3-functions" + fi + else + skipped "$lib-exports-no-sz3-functions (no shared ELF lib$lib here)" + fi +done + +# ---------------------------------------------------------------- 7. the filter as a component +mkdir -p comp +cat > comp/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(comp CXX) +find_package(SZ3 REQUIRED COMPONENTS hdf5sz3) +EOF +cmake -S comp -B comp/b -DCMAKE_PREFIX_PATH="$CMPFX" > comp/cfg.log 2>&1 +found=$? +if { [ "$HAVE_FILTER" = 1 ] && [ "$found" = 0 ]; } || { [ "$HAVE_FILTER" = 0 ] && [ "$found" != 0 ]; }; then + ok "hdf5sz3-component-found-exactly-when-installed" +else + bad "hdf5sz3-component-found-exactly-when-installed" "filter installed: $HAVE_FILTER, cmake exit: $found" \ + "$(tail -15 comp/cfg.log)" +fi + +# ---------------------------------------------------------------- 8. a versioned SONAME +if [ -f "$LIBDIR/libhdf5sz3.so" ] && command -v readelf > /dev/null 2>&1; then + readelf -d "$LIBDIR/libhdf5sz3.so" > soname.log 2>&1 + if grep -qE "SONAME.*libhdf5sz3\.so\.[0-9]" soname.log; then + ok "hdf5sz3-soname-carries-a-version" + else + bad "hdf5sz3-soname-carries-a-version" "$(grep SONAME soname.log)" + fi +else + skipped "hdf5sz3-soname-carries-a-version (no shared ELF libhdf5sz3 here)" +fi + +# ---------------------------------------------------------------- 9. OpenMP +# SZ3::SZ3 hands a consumer the OpenMP SZ3 was built with; SZ3::SZ3core and SZ3::hdf5sz3 do not. +if grep -qxE 'if\((ON|TRUE|1)\)' "$LIBDIR/cmake/SZ3/SZ3Config.cmake"; then +mkdir -p ompon +cat > ompon/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(ompon CXX) +find_package(SZ3 REQUIRED) +add_executable(ompon main.cpp) +target_link_libraries(ompon PRIVATE SZ3::SZ3) +EOF +cat > ompon/main.cpp <<'EOF' +#include +#ifndef _OPENMP +#error "SZ3 was built with OpenMP, and linking SZ3::SZ3 did not compile this consumer with it" +#endif +int main() { return 0; } +EOF +if cmake -S ompon -B ompon/b -DCMAKE_PREFIX_PATH="$CMPFX" > ompon/cfg.log 2>&1 && + cmake --build ompon/b --config Release > ompon/build.log 2>&1; then + ok "sz3-consumer-compiles-with-openmp" +else + bad "sz3-consumer-compiles-with-openmp" "$(tail -15 ompon/cfg.log ompon/build.log 2>/dev/null)" +fi +else +skip_all "this SZ3 was built without OpenMP" sz3-consumer-compiles-with-openmp +fi + +# The GROMACS shape. +mkdir -p ompoff +cat > ompoff/CMakeLists.txt <<'EOF' +cmake_minimum_required(VERSION 3.18) +project(ompoff CXX) +find_package(SZ3 REQUIRED) +add_executable(ompoff main.cpp) +target_link_libraries(ompoff PRIVATE SZ3::SZ3core) +if(TARGET SZ3::hdf5sz3) + target_link_libraries(ompoff PRIVATE SZ3::hdf5sz3) + target_compile_definitions(ompoff PRIVATE WITH_FILTER) +endif() +EOF +cat > ompoff/main.cpp <<'EOF' +#ifdef WITH_FILTER +#include +#endif +#include +#ifdef _OPENMP +#error "linking SZ3::SZ3core and SZ3::hdf5sz3 compiled this consumer with OpenMP" +#endif +int main() { +#ifdef WITH_FILTER + return H5Zregister(H5PLget_plugin_info()) < 0; +#else + return 0; +#endif +} +EOF +if cmake -S ompoff -B ompoff/b -DCMAKE_PREFIX_PATH="$CMPFX" > ompoff/cfg.log 2>&1 && + cmake --build ompoff/b --config Release > ompoff/build.log 2>&1; then + ok "sz3core-consumer-compiles-without-openmp" +else + bad "sz3core-consumer-compiles-without-openmp" "$(tail -15 ompoff/cfg.log ompoff/build.log 2>/dev/null)" +fi + +echo +echo " $pass passed, $fail failed, $skip skipped" + +# Raise this with the check it comes with. A section that stops early otherwise shows only as a +# smaller number at the bottom that nobody compares. +EXPECTED=20 +ran=$((pass + fail + skip)) +if [ "$ran" -ne "$EXPECTED" ]; then + echo " the suite accounted for $ran checks, not $EXPECTED" + exit 1 +fi +exit $((fail > 0)) diff --git a/tools/test/crossBuildDecode.sh b/tools/test/crossBuildDecode.sh new file mode 100755 index 00000000..34f476f5 --- /dev/null +++ b/tools/test/crossBuildDecode.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Data compressed by either of two builds (e.g. one with FMA, one without) must decompress to the same values +# in both. +# +# tools/test/crossBuildDecode.sh +# tools/test/crossBuildDecode.sh build build-fma tools/sz3/testfloat_8_8_128.dat -3 128 8 8 +set -u +A=$(cd "$1" && pwd)/tools/sz3/sz3 +B=$(cd "$2" && pwd)/tools/sz3/sz3 +INPUT=$(cd "$(dirname "$3")" && pwd)/$(basename "$3") +shift 3 +DIMS=("$@") +for f in "$A" "$B"; do [ -x "$f" ] || { echo "no sz3 binary at $f"; exit 1; }; done +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +cd "$WORK" || exit 1 +python3 -c "import array, sys; array.array('d', array.array('f', open(sys.argv[1], 'rb').read())).tofile(open('in.f64', 'wb'))" "$INPUT" +# ALGO_BIOMD takes coordinates {frames, atoms, 3}: 5 frames of 1000 rigid waters and a chain of 300 bonded atoms, in nm +python3 - <<'PY' +import array, math, random +random.seed(1) +def unit(v): + n = math.sqrt(sum(c * c for c in v)); return [c / n for c in v] +def cross(a, b): + return [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +def rvec(): + return unit([random.gauss(0, 1) for _ in range(3)]) +r, th = 0.09572, math.radians(104.52) +waters = [([random.uniform(0, 3) for _ in range(3)], rvec(), rvec()) for _ in range(1000)] +chain = [rvec() for _ in range(300)] +x = array.array('f') +for f in range(5): + p = [1.0, 1.0, 1.0] + for k, d in enumerate(chain): + p = [p[c] + (0.153 if k % 2 else 0.109) * d[c] for c in range(3)]; x.extend(p) + for o, u, w in waters: + v = unit(cross(u, w)) + x.extend(o); x.extend([o[c] + r * u[c] for c in range(3)]) + x.extend([o[c] + r * (math.cos(th) * u[c] + math.sin(th) * v[c]) for c in range(3)]) + waters = [([o[c] + 0.003 * random.gauss(0, 1) for c in range(3)], unit([u[c] + 0.05 * random.gauss(0, 1) for c in range(3)]), w) + for o, u, w in waters] + chain = [unit([d[c] + 0.03 * random.gauss(0, 1) for c in range(3)]) for d in chain] +x.tofile(open('coords.f32', 'wb')) +array.array('d', x).tofile(open('coords.f64', 'wb')) +PY +COORD_DIMS=(-3 3 3300 5) + +streams=0; fail=0 +while IFS='|' read -r name algo settings; do + for eb in 1e-2 1e-3 1e-4 1e-5 1e-6; do + printf '[GlobalSettings]\nCmprAlgo = %s\nErrorBoundMode = ABS\nAbsErrorBound = %s\n[AlgoSettings]\n%b' \ + "$algo" "$eb" "$settings" > c.ini + for t in f d; do + in=$INPUT; [ $t = d ] && in=in.f64 + dims=("${DIMS[@]}") + if [ "$algo" = ALGO_BIOMD ]; then in=coords.f$([ $t = d ] && echo 64 || echo 32); dims=("${COORD_DIMS[@]}"); fi + for w in A B; do + streams=$((streams + 1)) + rm -f s.sz a.out b.out + "${!w}" -$t -i "$in" -z s.sz "${dims[@]}" -c c.ini > /dev/null && + "$A" -$t -s s.sz -o a.out "${dims[@]}" > /dev/null && + "$B" -$t -s s.sz -o b.out "${dims[@]}" > /dev/null && + cmp -s a.out b.out || + { echo "MISMATCH $name eb=$eb type=$t written by $w"; fail=$((fail + 1)); } + done + done + done +done <<'LIST' +lorenzo_reg|ALGO_LORENZO_REG| +lorenzo2_reg|ALGO_LORENZO_REG|Lorenzo2ndOrder = YES\n +regression|ALGO_LORENZO_REG|Lorenzo = NO\n +interp_linear|ALGO_INTERP|InterpolationAlgo = INTERP_ALGO_LINEAR\n +interp_cubic|ALGO_INTERP|InterpolationAlgo = INTERP_ALGO_CUBIC\n +interp_lorenzo|ALGO_INTERP_LORENZO| +biomd|ALGO_BIOMD| +LIST + +echo "$fail of $streams streams decode differently" +[ "$streams" -eq 140 ] && [ "$fail" -eq 0 ] diff --git a/tools/test/data_format_digest.txt b/tools/test/data_format_digest.txt new file mode 100644 index 00000000..2aea349e --- /dev/null +++ b/tools/test/data_format_digest.txt @@ -0,0 +1,10 @@ +# SHA256 of tools/sz3/testfloat_8_8_128.dat compressed by a SZ3_USE_BUNDLED_ZSTD=ON build, as +# sz3 -f -i -z out -3 8 8 128 -M ABS 1e-3 +# +# The bundled Zstd, because the bytes depend on which Zstd compressed them and only it is pinned. +# +# A format change adds a row and bumps SZ3_DATA_VERSION. Only a re-vendored Zstd edits one. +# +# version sha256 +3.3.2 3171437b621e26d902271216e9cda5525e2232b9973917c324c0012baa054a1c +3.4.0 2353117ba820a0b2c77e059b67df98ba267c713f7829b5c4739f3fe3853a67ed diff --git a/tools/test/gromacsConsumerMatrix.sh b/tools/test/gromacsConsumerMatrix.sh new file mode 100755 index 00000000..1c6578cb --- /dev/null +++ b/tools/test/gromacsConsumerMatrix.sh @@ -0,0 +1,222 @@ +#!/bin/bash +# Every value of GROMACS's GMX_USE_SZ3, asserted against an installed SZ3. +# +# tools/test/gromacsConsumerMatrix.sh \ +# [extra cmake args...] +# +# The fifth argument says what the installed SZ3 does here: get found, not be installed at all, or +# decline because a dependency of its own cannot be resolved. Declining and being absent look the +# same to GROMACS, which is the point -- an SZ3 that ends the configure instead of declining +# breaks GMX_USE_SZ3=AUTO, whose contract is the fallback. They must not look the same to this +# script, so it asks the installed SZ3 itself before believing anything GROMACS says about it. +# +# Assert which library was wired in, never just that the configure exited 0. +set -u +HERE=$(cd "$(dirname "$0")" && pwd) +GMXSRC=$(cd "$1" && pwd) +WORK=$2 +SZ3_PREFIX=$3 +HDF5_PREFIX=$(cd "$4" && pwd) +EXPECT=$5 +shift 5 +mkdir -p "$WORK" +WORK=$(cd "$WORK" && pwd) + +case $EXPECT in +found) WIRED=external ;; +declined) WIRED=internal ;; +absent) WIRED=internal ;; +*) echo "unknown expectation $EXPECT: want found, absent or declined"; exit 1 ;; +esac +if [ "$EXPECT" = absent ] && [ "$SZ3_PREFIX" != "-" ]; then + echo "expecting no SZ3 to be installed, but $SZ3_PREFIX was given as one to find" + exit 1 +fi +if [ "$EXPECT" != absent ] && [ "$SZ3_PREFIX" = "-" ]; then + echo "expecting an installed SZ3 to be $EXPECT, with no prefix for it to be installed in" + exit 1 +fi + +PREFIX_PATH=$HDF5_PREFIX +SZ3_VERSION= +if [ "$SZ3_PREFIX" != "-" ]; then + SZ3_PREFIX=$(cd "$SZ3_PREFIX" && pwd) + PREFIX_PATH="$HDF5_PREFIX;$SZ3_PREFIX" + SZ3_VERSION=$(sed -n 's/^set(PACKAGE_VERSION "\(.*\)").*/\1/p' \ + "$SZ3_PREFIX"/lib*/cmake/SZ3/SZ3ConfigVersion.cmake 2>/dev/null | head -1) + # Required, not skipped: empty, the version assertion matches nothing and the check that + # INTERNAL ignores this prefix drops out silently, which reads as a pass. + if [ -z "$SZ3_VERSION" ]; then + echo "no version in $SZ3_PREFIX/lib*/cmake/SZ3/SZ3ConfigVersion.cmake" + exit 1 + fi +fi + +# The cheapest GROMACS that still generates the h5md link line. +# BUILD_TESTING is on only because h5md-test is the target whose link line names the library. +COMMON=(-DCMAKE_BUILD_TYPE=Release + -DGMX_SIMD=None + -DGMX_FFT_LIBRARY=fftpack + -DGMX_GPU=OFF + -DGMX_OPENMP=OFF + -DGMX_BUILD_OWN_FFTW=OFF + -DBUILD_TESTING=ON + -DGMX_USE_HDF5=ON + "-DCMAKE_PREFIX_PATH=$PREFIX_PATH") + +LINKTXT=src/gromacs/fileio/h5md/tests/CMakeFiles/h5md-test.dir/link.txt + +pass=0; fail=0 +ok() { echo "PASS $1"; pass=$((pass+1)); } +bad() { echo "FAIL $1"; shift; for l in "$@"; do echo " $l"; done; fail=$((fail+1)); } +# want +want() { if grep -qF "$2" "$3"; then ok "$1"; else bad "$1" "expected to find: $2" "got:" "$(grep -i sz3 "$3" | head -5)"; fi; } +notwant() { if grep -qF "$2" "$3"; then bad "$1" "did not expect: $2" "$(grep -nF "$2" "$3" | head -3)"; else ok "$1"; fi; } +# Which SZ3 a link token is: the installed one, or one GROMACS built in its own tree. A relative +# path is a build-tree library by construction, so only the install prefix has to be named here. +from_prefix() { while read -r t; do case $t in "$SZ3_PREFIX"/*) echo "$t";; esac; done; } +elsewhere() { while read -r t; do case $t in "$SZ3_PREFIX"/*) ;; *) echo "$t";; esac; done; } +oneline() { if [ -s "$1" ]; then tr '\n' ' ' < "$1" | sed 's/ *$//'; else printf '(none)'; fi; } + +# Every link-line check below rests on one question -- does this token link an SZ3 library -- +# so make the answer prove itself both ways before anything is asserted with it. +"$HERE"/sz3LinkTokens.sh --self-test || exit 1 + +EXTRA=("$@") +echo "=== GMX_USE_SZ3 matrix: $GMXSRC ===" +echo " hdf5 $HDF5_PREFIX" +echo " sz3 ${SZ3_PREFIX} ${SZ3_VERSION:+(version $SZ3_VERSION)}" +echo " expect $EXPECT, so GROMACS should wire in the $WIRED SZ3" +echo " extra ${EXTRA[*]:-none}" + +# GROMACS asks with find_package(SZ3 ... QUIET), which logs neither the answer nor the reason, so +# every expectation below rests on a premise no GROMACS log can confirm. A scenario that failed to +# arrange itself otherwise reads, ten lines later, like the defect it looks for. +mkdir -p "$WORK/probe" +cat > "$WORK/probe/CMakeLists.txt" <<'EOF' +cmake_minimum_required(VERSION 3.18) +# C and CXX, as GROMACS enables both: SZ3Config.cmake resolves OpenMP for whichever it is given. +project(sz3probe C CXX) +find_package(SZ3 QUIET) +if(SZ3_FOUND) + set(answer found) +elseif(SZ3_NOT_FOUND_MESSAGE) + set(answer declined) +elseif(SZ3_CONSIDERED_CONFIGS) + # An SZ3Config.cmake that was neither accepted nor declined -- too old a version, say. + set(answer rejected) +else() + set(answer absent) +endif() +string(STRIP "${answer} ${SZ3_NOT_FOUND_MESSAGE}" sz3_answer) +message(STATUS "answer: ${sz3_answer}") +if(NOT answer STREQUAL SZ3_EXPECTED_ANSWER) + message(FATAL_ERROR "the installed SZ3 answers ${answer}, not ${SZ3_EXPECTED_ANSWER}") +endif() +EOF +rm -rf "$WORK/probe/b" +cmake -S "$WORK/probe" -B "$WORK/probe/b" --no-warn-unused-cli \ + "-DCMAKE_PREFIX_PATH=$PREFIX_PATH" "-DSZ3_EXPECTED_ANSWER=$EXPECT" "${EXTRA[@]}" \ + > "$WORK/probe.log" 2>&1 +probe_rc=$? +echo " asked $(sed -n 's/^-- answer: //p' "$WORK/probe.log" | head -1)" +if [ "$probe_rc" != 0 ]; then + echo "this run's premise does not hold, so nothing below would mean what it says:" + tail -6 "$WORK/probe.log" | sed 's/^/ /' + exit 1 +fi + +for mode in EXTERNAL AUTO INTERNAL OFF; do + rm -rf "$WORK/b-$mode" + start=$(date +%s) + cmake -S "$GMXSRC" -B "$WORK/b-$mode" "${COMMON[@]}" "-DGMX_USE_SZ3=$mode" "${EXTRA[@]}" \ + > "$WORK/$mode.log" 2>&1 + rc=$? + echo "--- GMX_USE_SZ3=$mode: cmake exit $rc, $(( $(date +%s) - start ))s" + LINE=$WORK/b-$mode/$LINKTXT + : > "$WORK/$mode.link" + [ -f "$LINE" ] && "$HERE"/sz3LinkTokens.sh "$LINE" > "$WORK/$mode.link" + from_prefix < "$WORK/$mode.link" > "$WORK/$mode.installed" + elsewhere < "$WORK/$mode.link" > "$WORK/$mode.own" + + case "$mode:$WIRED" in + EXTERNAL:external) + if [ "$rc" = 0 ]; then ok "EXTERNAL configures"; else bad "EXTERNAL configures" "$(tail -15 "$WORK/$mode.log")"; fi + want "EXTERNAL reports the installed SZ3 $SZ3_VERSION" \ + "Found external SZ3 library (found version $SZ3_VERSION)" "$WORK/$mode.log" + if [ -s "$WORK/$mode.installed" ] && [ ! -s "$WORK/$mode.own" ]; then + ok "EXTERNAL links the installed libhdf5sz3 by path" + sed 's/^/ /' "$WORK/$mode.installed" + else + bad "EXTERNAL links the installed libhdf5sz3 by path" \ + "the unversioned hdf5sz3 target has to come out of the export, not as a bare -l" \ + "SZ3 libraries linked: $(oneline "$WORK/$mode.link")" + fi + ;; + EXTERNAL:internal) + # GROMACS's own error, not a crash inside SZ3Config.cmake: the caller asked for external. + if [ "$rc" = 0 ]; then + bad "EXTERNAL fails when no SZ3 can be used" "cmake exited 0" + else + ok "EXTERNAL fails when no SZ3 can be used" + fi + want "EXTERNAL fails with GROMACS's own message" \ + "Building with an external SZ3 library was selected" "$WORK/$mode.log" + ;; + AUTO:external) + if [ "$rc" = 0 ]; then ok "AUTO configures"; else bad "AUTO configures" "$(tail -15 "$WORK/$mode.log")"; fi + want "AUTO takes the installed SZ3" \ + "Found external SZ3 library (found version $SZ3_VERSION)" "$WORK/$mode.log" + notwant "AUTO does not also build the internal copy" "Using internal SZ3 library" "$WORK/$mode.log" + ;; + AUTO:internal) + if [ "$rc" = 0 ]; then + ok "AUTO still configures when SZ3 cannot be used" + else + bad "AUTO still configures when SZ3 cannot be used" \ + "AUTO's contract is the internal fallback; SZ3 must decline, not end the configure" \ + "$(tail -15 "$WORK/$mode.log")" + fi + want "AUTO falls back to the internal SZ3" "Using internal SZ3 library" "$WORK/$mode.log" + ;; + INTERNAL:*) + if [ "$rc" = 0 ]; then ok "INTERNAL configures"; else bad "INTERNAL configures" "$(tail -15 "$WORK/$mode.log")"; fi + want "INTERNAL uses GROMACS's own copy" "Using internal SZ3 library" "$WORK/$mode.log" + # Absence on its own would also be satisfied by a build that links no SZ3 whatsoever. + if [ -s "$WORK/$mode.own" ]; then + ok "INTERNAL links the SZ3 in GROMACS's own tree" + sed 's/^/ /' "$WORK/$mode.own" + else + bad "INTERNAL links the SZ3 in GROMACS's own tree" \ + "SZ3 libraries linked: $(oneline "$WORK/$mode.link")" + fi + # Only when there is an installed SZ3 to ignore: a check that cannot fail is not a check. + # Any mention of the prefix, not just a library from it: a -L or an rpath into it would + # still let the link, or the loaded binary, reach the copy INTERNAL is meant to skip. + if [ -n "$SZ3_VERSION" ]; then + if [ -f "$LINE" ] && grep -qF "$SZ3_PREFIX" "$LINE"; then + bad "INTERNAL ignores the installed SZ3" \ + "$(tr ' ' '\n' < "$LINE" | grep -F "$SZ3_PREFIX" | head -5)" + else + ok "INTERNAL ignores the installed SZ3" + fi + fi + ;; + OFF:*) + if [ "$rc" = 0 ]; then ok "OFF configures"; else bad "OFF configures" "$(tail -15 "$WORK/$mode.log")"; fi + if [ -s "$WORK/$mode.link" ]; then + bad "OFF links no SZ3 at all" "SZ3 libraries linked: $(oneline "$WORK/$mode.link")" + else + ok "OFF links no SZ3 at all" + fi + ;; + *) + # Unreachable while $WIRED is validated above; here so a mode added to the loop is not + # silently unasserted. + bad "unhandled GMX_USE_SZ3 value" "$mode:$WIRED" + ;; + esac +done + +echo "== $pass passed, $fail failed ==" +[ "$fail" = 0 ] diff --git a/tools/test/gromacsH5mdRoundtrip.sh b/tools/test/gromacsH5mdRoundtrip.sh new file mode 100755 index 00000000..5192090c --- /dev/null +++ b/tools/test/gromacsH5mdRoundtrip.sh @@ -0,0 +1,264 @@ +#!/bin/bash +# GROMACS writes an SZ3-compressed H5MD trajectory, and everything that reads it back is checked +# against the coordinates GROMACS handed the compressor. +# +# tools/test/gromacsH5mdRoundtrip.sh [ndec] +# +# ndec is GROMACS's output precision in decimal places; the error bound it asks SZ3 for is +# 1/(2*10^ndec) nm. Assert the numbers, never just that the exit was 0. +set -u +# Everything is resolved before the cd below, so the caller may pass relative paths. +GMX=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") +PLUGIN_DIR=$(cd "$2" && pwd) +H5BIN=$(cd "$3" && pwd) +TRAJ=$(cd "$(dirname "$4")" && pwd)/$(basename "$4") +TOPOL=$(cd "$(dirname "$5")" && pwd)/$(basename "$5") +WORK=$6 +NDEC=${7:-2} +# PYTHON only has to import numpy; nothing here needs HDF5 or GROMACS bindings. +PY=${PYTHON:-python3} +mkdir -p "$WORK" +WORK=$(cd "$WORK" && pwd) +cd "$WORK" || exit 1 + +DSET=/particles/system/position/value +NOPLUGIN=$WORK/no-such-plugin-dir + +pass=0; fail=0 +ok() { echo "PASS $1"; pass=$((pass+1)); } +bad() { echo "FAIL $1"; shift; for l in "$@"; do echo " $l"; done; fail=$((fail+1)); } +# want +want() { if grep -qF "$2" "$3"; then ok "$1"; else bad "$1" "expected to find: $2" "got:" "$(head -20 "$3")"; fi; } + +# Required, not skipped: a missing tool must never look like a passing check. +missing= +for tool in h5dump h5repack; do + command -v "$H5BIN/$tool" > /dev/null 2>&1 || missing="$missing $tool" +done +command -v "$GMX" > /dev/null 2>&1 || missing="$missing $(basename "$GMX")" +"$PY" -c 'import numpy' > /dev/null 2>&1 || missing="$missing python3-numpy" +if [ -n "$missing" ]; then + echo "required tools not found:$missing" + exit 1 +fi +# Required, not defaulted: empty, the version assertion below looks for "H5Z-SZ3-" and any version +# satisfies it, which is the one thing this pin exists to rule out. +if [ -z "${SZ3_EXPECT_VERSION:-}" ]; then + echo "set SZ3_EXPECT_VERSION to the version the filter has to report" + exit 1 +fi + +BOUND=$("$PY" -c "print(1.0 / (2.0 * 10 ** $NDEC))") +echo "=== $($GMX --version 2>/dev/null | grep -i 'GROMACS version' | head -1 | sed 's/ */ /g') ===" +echo " trajectory $TRAJ" +echo " error bound $BOUND nm (-ndec $NDEC)" + +# ------------------------------------------------------------------ write +# Two passes over the same frames with the same selection: the .trr carries the float32 +# coordinates GROMACS gave the compressor, so it is the reference the .h5md is judged against. +echo 0 | HDF5_PLUGIN_PATH=$PLUGIN_DIR "$GMX" trjconv -f "$TRAJ" -s "$TOPOL" -o lossy.h5md \ + -ndec "$NDEC" > write_lossy.log 2>&1 +lossy_rc=$? +echo 0 | "$GMX" trjconv -f "$TRAJ" -s "$TOPOL" -o ref.trr > write_ref.log 2>&1 +ref_rc=$? +if [ "$lossy_rc" = 0 ] && [ -s lossy.h5md ]; then + ok "gmx trjconv writes an SZ3-compressed .h5md" +else + bad "gmx trjconv writes an SZ3-compressed .h5md" "exit $lossy_rc" "$(tail -15 write_lossy.log)" + echo "== $pass passed, $fail failed ==" + exit 1 +fi +if [ "$ref_rc" = 0 ] && [ -s ref.trr ]; then + ok "gmx trjconv writes the uncompressed .trr reference" +else + bad "gmx trjconv writes the uncompressed .trr reference" "exit $ref_rc" "$(tail -15 write_ref.log)" + echo "== $pass passed, $fail failed ==" + exit 1 +fi + +# ------------------------------------------------------------------ the filter is really on it +# No plugin reachable: h5dump can still report the filter, and cannot silently decode instead. +HDF5_PLUGIN_PATH=$NOPLUGIN "$H5BIN/h5dump" -pH -d "$DSET" lossy.h5md > layout.txt 2>&1 +want "the position dataset carries filter 32024" "FILTER_ID 32024" layout.txt +# SZ3_EXPECT_VERSION pins which SZ3 did the compressing, for a consumer that vendors one. +want "the filter names itself as SZ3 ${SZ3_EXPECT_VERSION:-}" \ + "H5Z-SZ3-${SZ3_EXPECT_VERSION:-}" layout.txt +sed -n '/STORAGE_LAYOUT/,/FILTERS/p' layout.txt | sed 's/^/ /' + +# h5repack decodes every chunk through the plugin and writes the same trajectory with no filter. +HDF5_PLUGIN_PATH=$PLUGIN_DIR "$H5BIN/h5repack" -f NONE lossy.h5md plain.h5md > repack.log 2>&1 +repack_rc=$? +# h5repack exits 0 when it cannot decode a dataset, and drops it with a warning on stdout, so the +# exit status on its own says nothing. +repack_ok=yes +[ "$repack_rc" = 0 ] || repack_ok=no +[ -s plain.h5md ] || repack_ok=no +if grep -q "cannot be read" repack.log; then repack_ok=no; fi +if [ "$repack_ok" = yes ]; then + ok "h5repack decompresses the whole file through the plugin" +else + bad "h5repack decompresses the whole file through the plugin" "exit $repack_rc" "$(tail -10 repack.log)" +fi +LOSSY_BYTES=$(stat -c %s lossy.h5md) +PLAIN_BYTES=$(stat -c %s plain.h5md 2>/dev/null || echo 0) +if [ "$PLAIN_BYTES" -gt "$LOSSY_BYTES" ]; then + ok "the compressed file is smaller than the same trajectory uncompressed" +else + bad "the compressed file is smaller than the same trajectory uncompressed" \ + "compressed $LOSSY_BYTES bytes, uncompressed $PLAIN_BYTES bytes" +fi + +# ------------------------------------------------------------------ a reader with no GROMACS in it +cat > rd.c <<'EOF' +/* Reads the position dataset and writes it as raw float32. Links libhdf5 and nothing else, so the + SZ3 plugin is its only way to the coordinates. */ +#include +#include +#include +int main(int argc, char** argv) { + if (argc < 4) { fprintf(stderr, "usage: rd \n"); return 2; } + hid_t f = H5Fopen(argv[1], H5F_ACC_RDONLY, H5P_DEFAULT); + if (f < 0) { printf("OPEN FAILED\n"); return 2; } + hid_t d = H5Dopen2(f, argv[2], H5P_DEFAULT); + if (d < 0) { printf("DATASET NOT FOUND\n"); return 2; } + hid_t s = H5Dget_space(d); + hsize_t dims[3] = {0, 0, 0}; + int nd = H5Sget_simple_extent_dims(s, dims, NULL); + if (nd != 3) { printf("UNEXPECTED RANK %d\n", nd); return 2; } + size_t n = (size_t)dims[0] * dims[1] * dims[2]; + float* buf = (float*)malloc(n * sizeof(float)); + if (!buf) { printf("OUT OF MEMORY\n"); return 2; } + if (H5Dread(d, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) { + printf("READ FAILED\n"); return 1; + } + FILE* out = fopen(argv[3], "wb"); + if (!out || fwrite(buf, sizeof(float), n, out) != n) { printf("WRITE FAILED\n"); return 2; } + fclose(out); + printf("READ OK frames=%llu atoms=%llu dim=%llu\n", + (unsigned long long)dims[0], (unsigned long long)dims[1], (unsigned long long)dims[2]); + return 0; +} +EOF +# Built against the HDF5 whose tools are being driven, so the reader and h5dump agree on a version. +H5ROOT=$(cd "$H5BIN/.." && pwd) +if [ -f "$H5ROOT/include/hdf5.h" ]; then + cc -O1 -o rd rd.c -I"$H5ROOT/include" -L"$H5ROOT/lib" -lhdf5 -Wl,-rpath,"$H5ROOT/lib" \ + > build_rd.log 2>&1 +else + cc -O1 -o rd rd.c $(pkg-config --cflags --libs hdf5 2>/dev/null || echo "-lhdf5") > build_rd.log 2>&1 +fi +if [ -x ./rd ]; then + ok "the plain HDF5 reader builds" +else + bad "the plain HDF5 reader builds" "$(tail -10 build_rd.log)" + echo "== $pass passed, $fail failed ==" + exit 1 +fi +ldd ./rd > rd.ldd 2>&1 +# The dependency names, not the directories they resolve in: a reader that found its HDF5 under +# a directory called sz3 or gromacs is still a reader with neither of them linked into it. +deps=$(awk '{print $1}' rd.ldd | sed 's|.*/||') +banned=$(echo "$deps" | grep -Ei 'gromacs|sz3' | tr '\n' ' ') +if ! echo "$deps" | grep -q '^libhdf5'; then + # ldd's own error text matches nothing below, which would read as a pass. + bad "the reader links no GROMACS and no SZ3" "ldd listed no libhdf5:" "$(cat rd.ldd)" +elif [ -n "$banned" ]; then + bad "the reader links no GROMACS and no SZ3" "$banned" +else + ok "the reader links no GROMACS and no SZ3" +fi + +HDF5_PLUGIN_PATH=$PLUGIN_DIR ./rd lossy.h5md "$DSET" decoded.f32 > read_ok.log 2>&1 +want "the reader decodes the coordinates through the plugin" "READ OK" read_ok.log +sed 's/^/ /' read_ok.log + +# The repacked copy carries no filter, so this read goes nowhere near SZ3 and is the second, +# independent decode of the same chunks. +./rd plain.h5md "$DSET" repacked.f32 > read_plain.log 2>&1 +want "the reader reads the repacked copy" "READ OK" read_plain.log +if cmp -s decoded.f32 repacked.f32; then + ok "both decodes of the trajectory agree bit for bit" +else + bad "both decodes of the trajectory agree bit for bit" \ + "the plugin and h5repack disagree about what the file holds" +fi + +# The same read with nowhere to find the plugin: HDF5 cannot decode, and must say so. +HDF5_PLUGIN_PATH=$NOPLUGIN ./rd lossy.h5md "$DSET" /dev/null > read_noplugin.log 2>&1 +noplugin_rc=$? +if [ "$noplugin_rc" = 0 ]; then + bad "without the plugin the read fails" "the read succeeded, so the file was not SZ3-compressed" +else + ok "without the plugin the read fails" +fi + +# ------------------------------------------------------------------ the numbers +"$PY" - "$NDEC" "$LOSSY_BYTES" "$PLAIN_BYTES" <<'PY' > compare.txt 2>&1 +import struct, sys +import numpy as np + +ndec, lossy_bytes, plain_bytes = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]) + +# GROMACS computes the bound in its own `real`, so the number SZ3 is given is the float32 one. +bound = float(np.float32(1.0 / (2.0 * 10 ** ndec))) + +# TRR is XDR big-endian: magic, two string lengths, the block sizes, natoms, step, nre, t, lambda, +# then the box and the coordinates. Nothing here is GROMACS code. +with open("ref.trr", "rb") as fh: + blob = fh.read() +if struct.unpack_from(">i", blob, 0)[0] != 1993: + print("TRR MAGIC MISMATCH"); raise SystemExit(1) +strlen = struct.unpack_from(">i", blob, 8)[0] +off = 12 + ((strlen + 3) // 4) * 4 +(box_size, _vir, _pres, _top, _sym, x_size, v_size, f_size, natoms) = struct.unpack_from(">9i", blob, off + 8) +off += 4 * 13 +_t, _lam = struct.unpack_from(">2f", blob, off) +header = off + 8 +frame_bytes = header + box_size + x_size + v_size + f_size +if len(blob) % frame_bytes != 0: + print("TRR FRAME SIZE MISMATCH", len(blob), frame_bytes); raise SystemExit(1) +nframes = len(blob) // frame_bytes +ref = np.empty((nframes, natoms, 3), dtype=">f4") +for i in range(nframes): + start = i * frame_bytes + header + box_size + ref[i] = np.frombuffer(blob, dtype=">f4", count=natoms * 3, offset=start).reshape(natoms, 3) +ref = ref.astype(" [ ... ]") + print("Example: python script.py ALGO_LORENZO_REG input.fp32 100 100 100") + sys.exit(1) + + # Input Arguments + cmpr_algo = sys.argv[1] + raw_file = sys.argv[2] + shape_args = sys.argv[3:] + + # Fixed Parameters + h5_dataset_name = 'test' # Fixed dataset name + # output_dir = './output_hdf5' # Fixed output directory + output_dir = tempfile.mkdtemp(prefix='output_hdf5_') + + # Parse Shape + shape = parse_shape(shape_args) + + # Define output file paths based on output_dir and input file name + base_name = os.path.splitext(os.path.basename(raw_file))[0] + original_h5 = os.path.join(output_dir, f"{base_name}_original.h5") + + # Ensure output directory exists + os.makedirs(output_dir, exist_ok=True) + + # Check for h5repack or h5repack-shared + repack_cmd = check_h5repack() + + # Step 1: Read raw data + data = read_raw_fp32(raw_file, shape) + + # Step 2: Convert to HDF5 + write_hdf5(data, original_h5, h5_dataset_name) + + passed_all_cases = True + # Iterate over each error bound and perform compression, decompression, and comparison + for bound in error_bounds: + print("\n" + "="*50) + print(f"Testing {raw_file} with algo = {cmpr_algo} AbsErrorBound = {bound}") + print("="*50) + + # Create comp.conf with the current bound + comp_conf_path = os.path.join(output_dir, f"comp_{format_bound(bound)}.conf") + create_comp_conf(comp_conf_path, cmpr_algo, bound) + + # Get Compression Arguments by calling external executable + compression = get_compression_args(comp_conf_path) + + # Define compressed and decompressed file paths for the current bound + bound_str = format_bound(bound) + compressed_h5 = os.path.join(output_dir, f"{base_name}_compressed_{bound_str}.h5") + decompressed_h5 = os.path.join(output_dir, f"{base_name}_decompressed_{bound_str}.h5") + + # Apply compression + apply_h5repack(repack_cmd, original_h5, compressed_h5, compression=compression) + + # Decompress the compressed HDF5 + apply_h5repack(repack_cmd, compressed_h5, decompressed_h5, compression='NONE') + + # Compare original and decompressed HDF5 files + max_error = compare_hdf5(original_h5, decompressed_h5, h5_dataset_name) + + # Verify that max_error <= bound * 1.1 + if max_error <= bound * 1.5: + result = "PASS" + else: + result = "FAIL" + passed_all_cases = False + + print(f"Test Result for AbsErrorBound = {bound}: {result}") + + # Clean up intermediate files + try: + rmtree(output_dir) + print(f"Cleaned up intermediate folder") + except Exception as e: + print(f"Error removing intermediate folder: {e}") + + if passed_all_cases: + print(f"\nAll tests passed successfully for {raw_file}.") + else: + print(f"\nSome tests failed for {raw_file}. Please check the results.") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/tools/test/integration/deprecated/test_sz3.py b/tools/test/integration/deprecated/test_sz3.py new file mode 100644 index 00000000..991f017d --- /dev/null +++ b/tools/test/integration/deprecated/test_sz3.py @@ -0,0 +1,146 @@ +import subprocess +import re +import os +from pathlib import Path +import sys + +HOME = str(Path.home()) + + +def run_command(cmd): + """Run a command and return stdout and stderr.""" + result = subprocess.run(cmd, shell=True, capture_output=True, text=True) + return result.stdout, result.stderr + + +def parse_errors(output): + """ + Parse the decompression output for maximum errors. + Expected output lines: + Max absolute error = + Max relative error = + Returns (abs_error, rel_error) as floats (or None if not found). + """ + abs_err = None + rel_err = None + abs_match = re.search(r"Max absolute error\s*=\s*([\deE\.\+-]+)", output) + if abs_match: + abs_err = float(abs_match.group(1)) + rel_match = re.search(r"Max relative error\s*=\s*([\deE\.\+-]+)", output) + if rel_match: + rel_err = float(rel_match.group(1)) + return abs_err, rel_err + + +def test_file(sz3_path, file_path, dims, mode, error_bound): + """ + Run the SZ3 command with the given file, reversed dimensions, error control mode, and error bound. + - file_path: path to the original binary file. + - dims: a list of dimensions (e.g. [100, 500, 500] from the test file). + - mode: either "ABS" or "REL". + - error_bound: error bound value to test. + + Returns: (abs_err, rel_err) from the output. + """ + # Derive filename for the decompressed file + base = os.path.splitext(os.path.basename(file_path))[0] + decompressed_file = "{}.{}.{}.sz.out".format(base, mode, error_bound) + + # Reverse the dimensions order for SZ3 (e.g., 100 500 500 becomes 500 500 100) + dims_str = " ".join(str(d) for d in dims[::-1]) + + # Build the single command that compresses and decompresses the file. + # Note: no -z is provided, only -i (input) and -o (decompressed output). + cmd = f"{sz3_path} -f -i {file_path} -o {decompressed_file} -3 {dims_str} -M {mode} {error_bound} -a" + + print("Running command:") + print(cmd) + stdout, stderr = run_command(cmd) + + if stderr: + print("stderr:", stderr) + + print("Command output:") + print(stdout) + + # Parse error values from the output + abs_err, rel_err = parse_errors(stdout) + + if os.path.exists(decompressed_file): + os.remove(decompressed_file) + + return abs_err, rel_err + +def load_test_files(file_list_path): + """ + Read the list of test files from the provided text file. + Each line should have: ... + Returns a list of tuples: (file_path, [dim1, dim2, ...]) + """ + test_files = [] + with open(file_list_path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + file_path = parts[0].replace('~', HOME) + try: + dims = [int(x) for x in parts[1:]] + except ValueError: + print(f"Error parsing dimensions for line: {line}") + continue + test_files.append((file_path, dims)) + return test_files + + +def main(): + if len(sys.argv) > 2: + sz3_path = sys.argv[1] + file_list_path = sys.argv[2] + else: + print("Usage: python test_sz3.py ") + exit() + + test_files = load_test_files(file_list_path) + + # Error bounds to test for each mode + error_tests = { + "ABS": [1, 1e-3, 1e-5, 1e-7], + "REL": [1e-2, 1e-4, 1e-6] + } + + all_passed = True + for file_path, dims in test_files: + if not os.path.exists(file_path): + print(f"Test file {file_path} does not exist, skipping.") + continue + for mode, bounds in error_tests.items(): + for bound in bounds: + print("=" * 60) + print(f"Testing file: {file_path}\nMode: {mode}\nError Bound: {bound}") + abs_err, rel_err = test_file(sz3_path, file_path, dims, mode, bound) + + # Check results depending on the mode + if mode == "ABS": + if abs_err is not None and abs_err <= bound: + print(f"PASS: Max absolute error {abs_err} is within bound {bound}") + else: + print(f"FAIL: Max absolute error {abs_err} exceeds bound {bound}") + all_passed = False + elif mode == "REL": + if rel_err is not None and rel_err <= bound: + print(f"PASS: Max relative error {rel_err} is within bound {bound}") + else: + print(f"FAIL: Max relative error {rel_err} exceeds bound {bound}") + all_passed = False + print("=" * 60 + "\n") + + if all_passed: + print("All tests passed.") + else: + print("Some tests failed.") + + +if __name__ == "__main__": + main() diff --git a/tools/test/integration/fetch_md_trajectory.py b/tools/test/integration/fetch_md_trajectory.py new file mode 100644 index 00000000..4741e84f --- /dev/null +++ b/tools/test/integration/fetch_md_trajectory.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Turn a published MD trajectory into the raw float32 arrays the integration tests read. + +The SDRBench fields the suite already covers are 1D and 2D, so the trajectory layout +ALGO_BIOMD was written for -- {frames, atoms, xyz} -- is never exercised on molecular +dynamics data. These are the MDAnalysisData benchmark trajectories, downloaded from the +figshare files that package points at and converted here, which keeps the test data to +stable checksummed URLs rather than a Python package's release cadence. + +Positions come out in whatever units MDAnalysis reports, which is Angstrom for every reader +used here, so an absolute error bound means the same thing across all of them. The XTC sets +arrive already quantized to GROMACS's own 0.001 nm grid, so below an error bound of 0.01 the +compressor reproduces them nearly exactly; the DCD and NetCDF sets carry full float precision. + +Usage: + python3 fetch_md_trajectory.py --list + python3 fetch_md_trajectory.py [--fields xyz,x,y,z] [--max-frames N] +""" + +import argparse +import hashlib +import os +import random +import time + +import numpy as np +import requests + +FIGSHARE = "https://ndownloader.figshare.com/files/" + +# topology and trajectory for each dataset, as (filename, figshare file id, sha256). +DATASETS = { + "adk-equilibrium": { + "topology": ("adk4AKE.psf", "8672230", + "1aa947d58fb41b6805dc1e7be4dbe65c6a8f4690f0bd7fc2ae03e7bd437085f4"), + "trajectory": ("1ake_007-nowater-core-dt240ps.dcd", "8672074", + "598fcbcfcc425f6eafbe9997238320fcacc6a4613ecce061e1521732bab734bf"), + "shape": (4187, 3341), + }, + "ifabp-water": { + "topology": ("ifabp_water.psf", "12980639", + "ba40714318aabec537015dc550fe5bd5ac1ac0b853f5abdd2f0ae63af9cfcafa"), + "trajectory": ("rmsfit_ifabp_water_1.dcd", "12980642", + "cebb48e58015abc8ff2f5bb7ba3eb7a289047f256351a8252bf1f29f9aaacf0e"), + "shape": (500, 12445), + }, + "membrane-peptide": { + "topology": ("memb_pept.tpr", "14993171", + "677a3ae55e35c24f37f2610eafa92d19285d1774731d6ffb9a99dfde39b8c437"), + "trajectory": ("memb_pept.xtc", "14993174", + "f9bdfee4e1aa69ccfeef21cb74703202f6728f514543c4125382bd5250773eb7"), + "shape": (1001, 18727), + }, + "cg-fiber": { + "topology": ("126chains.psf", "13374146", + "3ddb654b68549ac2ad5107a4282899f41fad233d09ea572446031711af4e57da"), + "trajectory": ("126chains.dcd", "13375838", + "e0b47d422f31ec209ea810edcf6cf3830da04bb2e1540f520477c27f4433d849"), + "shape": (2221, 8364), + }, + "peg-1chain": { + "topology": ("PEG.prmtop", "13532462", + "2d7955b9a8cb6e008171e0c5a1c31e3e458246ea3ee7302281eafefafa7cede9"), + "trajectory": ("PEG_03_prod.nc", "13532465", + "b978714ec2f93d1cbe99564cb257959f0cb38872359aa745c8eba720a7d85225"), + "shape": (50, 13406), + }, + "nhaa-equilibrium": { + "topology": ("NhaA_non_water.gro", "13222709", + "ae42f4cfcfe312476f9e5121fe47764a11aff962197799671c0c5a8f83637420"), + "trajectory": ("NhaA_non_water.xtc", "13222712", + "c9ab7ba8c9c271d535cfadebc33da1d90fbf00d9a01f48afedd0f7a703128eaf"), + "shape": (5001, 60702), + }, + "yiip-equilibrium": { + "topology": ("YiiP_system.pdb", "15286808", + "3c2b96bbd2f95105e1a4f37140132ee073a947df8fe209a8170f09ca5b73e6cf"), + "trajectory": ("YiiP_system_9ns_center.xtc", "15285461", + "97f6a93acc1e330915338b290625d81d84928a7e4c1e5aed63d209881fbe268b"), + "shape": (901, 111815), + }, +} + + +def download(file_id, filename, sha256, into): + """Fetch one figshare file, reusing it when it is already there and intact.""" + target = os.path.join(into, filename) + if os.path.exists(target) and file_digest(target) == sha256: + print(f"{filename} already present") + return target + + url = FIGSHARE + file_id + for attempt in range(5): + try: + print(f"Downloading {filename} from {url}") + with requests.get(url, stream=True, timeout=60) as response: + response.raise_for_status() + with open(target, "wb") as out: + for chunk in response.iter_content(chunk_size=1 << 20): + out.write(chunk) + # A connection that drops mid-transfer often closes cleanly, so the checksum rather + # than an exception is what says the file is short. Both are worth retrying. + digest = file_digest(target) + if digest == sha256: + return target + raise OSError(f"expected sha256 {sha256}, got {digest}") + except (requests.exceptions.RequestException, OSError) as error: + if os.path.exists(target): + os.remove(target) + if attempt == 4: + raise SystemExit(f"{filename}: {error}") + print(f"Download failed ({error}), retrying") + time.sleep(random.uniform(5, 20)) + raise SystemExit(f"{filename}: exhausted retries") + + +def file_digest(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def convert(topology, trajectory, output_dir, expected_shape, fields, max_frames=0): + """Write xyz.f32 as {frames, atoms, 3}, and one 2D {frames, atoms} file per named coordinate. + + Each coordinate file is a third the size of xyz.f32, so a large trajectory is usually asked + for one field at a time to stay inside a runner's disk. + """ + import MDAnalysis # imported here so --list works without it + + universe = MDAnalysis.Universe(topology, trajectory) + atoms = len(universe.atoms) + frames = len(universe.trajectory) + print(f"{frames} frames, {atoms} atoms") + if expected_shape and (frames, atoms) != tuple(expected_shape): + raise SystemExit(f"expected {expected_shape} frames x atoms, found {(frames, atoms)}") + if max_frames and max_frames < frames: + print(f"writing the leading {max_frames} frames") + frames = max_frames + + paths = {name: os.path.join(output_dir, f"{name}.f32") for name in fields} + handles = {} + try: + for name, path in paths.items(): + handles[name] = open(path, "wb") + for step, _ in enumerate(universe.trajectory): + if step == frames: + break + positions = universe.atoms.positions.astype(np.float32, copy=False) + if "xyz" in handles: + handles["xyz"].write(positions.tobytes()) + for axis, name in enumerate(("x", "y", "z")): + if name in handles: + handles[name].write(np.ascontiguousarray(positions[:, axis]).tobytes()) + if step % 500 == 0: + print(f" frame {step}/{frames}") + finally: + for handle in handles.values(): + handle.close() + + for name, path in paths.items(): + dims = [frames, atoms, 3] if name == "xyz" else [frames, atoms] + print(f" {os.path.basename(path)}: dims {dims}, {os.path.getsize(path)} bytes") + return frames, atoms + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dataset", nargs="?", help="one of: " + ", ".join(sorted(DATASETS))) + parser.add_argument("output_dir", nargs="?") + parser.add_argument("--keep-downloads", action="store_true", + help="keep the topology and trajectory files after converting") + parser.add_argument("--fields", default="xyz,x,y,z", + help="which of xyz,x,y,z to write (default all)") + parser.add_argument("--max-frames", type=int, default=0, + help="write only the leading N frames (default: all)") + parser.add_argument("--list", action="store_true", help="print the dataset names and exit") + args = parser.parse_args() + + if args.list: + for name, spec in sorted(DATASETS.items()): + frames, atoms = spec["shape"] + print(f"{name:<20} {frames} frames x {atoms} atoms") + return + + if not args.dataset or not args.output_dir: + parser.error("dataset and output_dir are required") + if args.dataset not in DATASETS: + parser.error(f"unknown dataset {args.dataset}; try --list") + + spec = DATASETS[args.dataset] + os.makedirs(args.output_dir, exist_ok=True) + + sources = [download(file_id, filename, checksum, args.output_dir) + for filename, file_id, checksum in (spec["topology"], spec["trajectory"])] + fields = [f for f in args.fields.split(",") if f] + unknown = [f for f in fields if f not in ("xyz", "x", "y", "z")] + if unknown: + parser.error(f"unknown fields {unknown}") + convert(sources[0], sources[1], args.output_dir, spec["shape"], fields, args.max_frames) + + if not args.keep_downloads: + for path in sources: + os.remove(path) + print(f"Removed {os.path.basename(path)}") + + +if __name__ == "__main__": + main() diff --git a/tools/test/integration/integration_test_driver.py b/tools/test/integration/integration_test_driver.py new file mode 100644 index 00000000..ace876a0 --- /dev/null +++ b/tools/test/integration/integration_test_driver.py @@ -0,0 +1,309 @@ +import csv +import subprocess +import os +import sys +import json +import requests +import tarfile +import tempfile +import shutil +import time +import random + +def get_tmpdir(): + candidates = ['/scratch', '/var/tmp', '/tmp'] + for cand in candidates: + if os.path.exists(cand): + try: + test_dir = os.path.join(cand, 'sz3_test_write') + os.makedirs(test_dir, exist_ok=True) + os.rmdir(test_dir) + return cand + except OSError: + pass + return tempfile.gettempdir() + +def run_test(cmd, description): + print('\n\n', "="*80) + print(description) + print("command:", " ".join(cmd)) + result = subprocess.run(cmd, capture_output=True, text=True) + print(result.stdout) + + if result.stderr: + print("STDERR:", result.stderr) + if result.returncode == 0: + print("PASS") + print("="*80) + return True, result.stdout + else: + print("FAIL") + print("="*80) + return False, result.stdout + + +BASELINE_TOLERANCE = 0.05 + + +def baseline_key(dataset, field, algo, eb, harness): + return f"{dataset}/{field}|{algo}|{eb}|{harness}" + + +def load_baseline(path): + """Compressed sizes a previous run produced. Absent means nothing is checked.""" + try: + with open(path) as handle: + return {k: v for k, v in json.load(handle).items() if not k.startswith("_")} + except FileNotFoundError: + print(f"No compression baseline at {path}; sizes will be reported, not checked") + return {} + + +def check_against_baseline(baseline, row): + """Compression is deterministic, so a size that moved means the output moved. + + Returns True when the case is within tolerance or has no baseline to compare against. + """ + key = baseline_key(row["dataset"], row["field"], row["algo"], row["error_bound"], row["harness"]) + expected = baseline.get(key) + if expected is None or not row["bytes"]: + if expected is None: + print(f"BASELINE new {key} = {row['bytes']}") + return True + actual = int(row["bytes"]) + drift = actual / expected - 1 + if abs(drift) <= BASELINE_TOLERANCE: + print(f"BASELINE ok {key}: {actual} vs {expected} ({drift:+.2%})") + return True + print(f"BASELINE FAIL {key}: {actual} vs {expected} ({drift:+.2%}), " + f"outside +/-{BASELINE_TOLERANCE:.0%}") + return False + + +def collect_metrics(dataset, field, algo, eb, harness, output): + """Every METRICS line a test printed. The HDF5 test prints one per chunk mode.""" + row = {"dataset": dataset, "field": field, "algo": algo, "error_bound": eb, "harness": harness, + "bytes": "", "ratio": "", "compress_s": "", "decompress_s": "", "max_error": ""} + rows = [] + for line in (output or "").splitlines(): + if not line.startswith("METRICS "): + continue + found = dict(row) + for pair in line[len("METRICS "):].split(): + key, _, value = pair.partition("=") + if key in found: + found[key] = value + elif key == "chunk": + found["harness"] = f"{harness}:{value}" + rows.append(found) + return rows + + +def write_metrics(rows, path): + """A ratio and a timing per case, so a regression in either is visible between runs.""" + if not rows: + return + with open(path, "w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + print(f"Wrote {len(rows)} rows of compression metrics to {path}") + + +def prepare_dataset(path, dataset_dir, dataset_info=None): + """ + Prepares the dataset: "mdtraj:" fetches and converts a published MD trajectory, + an http path is downloaded and extracted, a local path is copied. + Returns the actual directory that directly containing the files. + """ + if path.startswith('mdtraj:'): + # A published trajectory, which fetch_md_trajectory.py downloads and converts to the raw + # arrays every other dataset here already is. Only the fields this run asks for are + # written, because a whole trajectory in every layout does not fit a runner's disk. + os.makedirs(dataset_dir, exist_ok=True) + wanted = sorted({f.rsplit('.', 1)[0] for f in (dataset_info or {}).get("fields", {})}) + script = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fetch_md_trajectory.py') + cmd = [sys.executable, script, path.split(':', 1)[1], dataset_dir] + if wanted: + cmd += ['--fields', ','.join(wanted)] + if (dataset_info or {}).get("max_frames"): + cmd += ['--max-frames', str(dataset_info["max_frames"])] + subprocess.run(cmd, check=True) + return dataset_dir + + if path.startswith('http'): + # Download and extract + if not os.path.exists(dataset_dir): + os.makedirs(dataset_dir) + + tar_filename = os.path.join(dataset_dir, os.path.basename(path)) + + if not os.path.exists(tar_filename): + print(f"Downloading {path} to {tar_filename}") + for attempt in range(5): + try: + with requests.get(path, stream=True) as r: + r.raise_for_status() + with open(tar_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + break # Success + except requests.exceptions.ConnectionError as e: + if attempt < 4: + print(f"Download failed (attempt {attempt+1}/5), retrying in 5 seconds...") + time.sleep(random.uniform(30, 60)) + else: + raise + else: + print(f"{tar_filename} already exists. Skipping download.") + + print(f"Extracting {tar_filename} to {dataset_dir}") + with tarfile.open(tar_filename, 'r:gz') as tar_ref: + tar_ref.extractall(dataset_dir, filter='data') + + # Delete the tar file to save space if running in GitHub CI + if os.getenv('GITHUB_ACTIONS') == 'true': + os.remove(tar_filename) + print(f"Deleted {tar_filename} to save space in CI") + + # Find the data directory - if there's a single subdirectory, use it + subdirs = [d for d in os.listdir(dataset_dir) if os.path.isdir(os.path.join(dataset_dir, d))] + if len(subdirs) == 1: + return os.path.join(dataset_dir, subdirs[0]) + else: + return dataset_dir + else: + # Local path - assume it's a directory + if os.path.isdir(path): + if not os.path.exists(dataset_dir): + os.makedirs(dataset_dir) + if dataset_info: + for field in dataset_info["fields"].keys(): + src = os.path.join(path, field) + dst = os.path.join(dataset_dir, field) + if os.path.isfile(src): + shutil.copy2(src, dst) + else: + print(f"Warning: field file {src} not found") + else: + shutil.copytree(path, dataset_dir) + return dataset_dir + return dataset_dir + + +def main(): + if len(sys.argv) > 1: + datasets_json = sys.argv[1] + else: + print("No datasets.json provided ") + sys.exit(1) + + try: + with open(datasets_json, 'r') as f: + datasets = json.load(f) + except FileNotFoundError: + print(f"Error: Datasets file '{datasets_json}' not found.") + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in '{datasets_json}': {e}") + sys.exit(1) + + if len(sys.argv) > 2: + # "name" runs the whole dataset, "name:a.f32,b.f32" runs those fields of it, so a dataset + # too slow to be one job can be spread over several. + selected_dataset, _, selected_fields = sys.argv[2].partition(":") + if selected_dataset not in datasets: + print(f"Dataset {selected_dataset} not found in {datasets_json}") + sys.exit(1) + dataset_info = datasets[selected_dataset] + if selected_fields: + wanted = [f for f in selected_fields.split(",") if f] + missing = [f for f in wanted if f not in dataset_info["fields"]] + if missing: + print(f"Fields {missing} not found in dataset {selected_dataset}") + sys.exit(1) + dataset_info = dict(dataset_info) + dataset_info["fields"] = {f: dataset_info["fields"][f] for f in wanted} + datasets = {selected_dataset: dataset_info} + + script_dir = os.path.dirname(os.path.abspath(__file__)) + project_source_dir = os.path.abspath(os.path.join(script_dir, "..", "..", "..")) + + test_dir = os.path.join(get_tmpdir(), "sz3_integration_test") + data_dir = os.path.join(test_dir, "data") + build_dir = os.path.join(test_dir, "build") + + for d in [test_dir, data_dir]: + os.makedirs(d, exist_ok=True) + + original_wd = os.getcwd() + os.chdir(test_dir) + + # Build the project + os.makedirs(build_dir, exist_ok=True) + subprocess.run(["cmake", project_source_dir, "-DBUILD_H5Z_FILTER=ON"], cwd=build_dir, check=True) + subprocess.run(["cmake", "--build", ".", "--", "-j"], cwd=build_dir, check=True) + sz3_executable_path = os.path.join(build_dir, "tools", "sz3") + h5_plugin_path = os.path.join(build_dir, "tools", "H5Z-SZ3") + + error_bounds = [1e-1, 1e-2, 1e-3, 1e-4] + algorithms = ["ALGO_INTERP_LORENZO", "ALGO_LORENZO_REG", "ALGO_BIOMD"] + + results = [] + metrics = [] + baseline = load_baseline(os.path.join(script_dir, "compression_baseline.json")) + + try: + run_datasets(datasets, data_dir, script_dir, sz3_executable_path, h5_plugin_path, + algorithms, error_bounds, baseline, results, metrics) + finally: + write_metrics(metrics, os.path.join(original_wd, "integration_metrics.csv")) + + # Summary + total_tests = len(results) + passed = sum(results) + failed = total_tests - passed + print(f"\nSummary: {passed}/{total_tests} tests passed, {failed} failed.") + if failed > 0: + print("Some tests failed. Exiting with error.") + sys.exit(1) + + os.chdir(original_wd) + + +def run_datasets(datasets, data_dir, script_dir, sz3_executable_path, h5_plugin_path, + algorithms, error_bounds, baseline, results, metrics): + for dataset_name, dataset_info in datasets.items(): + dataset_dir = os.path.join(data_dir, dataset_name) + actual_data_dir = prepare_dataset(dataset_info["path"], dataset_dir, dataset_info) + + for field, field_info in dataset_info["fields"].items(): + dims = field_info["dims"] + dtype = field_info.get("dtype", "float32") + + for algo in algorithms: + for eb in error_bounds: + data_file = os.path.join(actual_data_dir, field) + if not os.path.isfile(data_file): + print(f"Data file {data_file} does not exist. Skipping") + continue + + # Call H5 test + cmd = [sys.executable, os.path.join(script_dir, "test_h5_filter.py"), h5_plugin_path, algo, str(eb), data_file, dtype] + [str(d) for d in dims] + passed, output = run_test(cmd, f"Testing HDF5 {algo} {eb} on {dataset_name}/{field}") + results.append(passed) + metrics.extend(collect_metrics(dataset_name, field, algo, eb, "hdf5", output)) + + # Call SZ3 test + cmd = [sys.executable, os.path.join(script_dir, "test_sz3_executable.py"), sz3_executable_path, algo, str(eb), data_file, dtype] + [str(d) for d in dims] + passed, output = run_test(cmd, f"Testing SZ3 EXE {algo} {eb} on {dataset_name}/{field}") + rows = collect_metrics(dataset_name, field, algo, eb, "cli", output) + metrics.extend(rows) + results.append(passed and all(check_against_baseline(baseline, r) for r in rows)) + + if os.getenv('GITHUB_ACTIONS') == 'true': + shutil.rmtree(dataset_dir) + + +if __name__ == "__main__": + main() diff --git a/tools/test/integration/test_h5_filter.py b/tools/test/integration/test_h5_filter.py new file mode 100644 index 00000000..d2aaa9be --- /dev/null +++ b/tools/test/integration/test_h5_filter.py @@ -0,0 +1,255 @@ +import numpy as np +import os +import re +import sys +from shutil import rmtree +import tempfile + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../H5Z-SZ3/tools')) +from cdvalueHelper import SZ3 + +if len(sys.argv) > 1: + os.environ["HDF5_PLUGIN_PATH"] = sys.argv[1] +import h5py + +if not h5py.h5z.filter_avail(32024): + print("SZ3 filter not available") + sys.exit(1) + + +def get_compression_args(cmpr_algo, bound): + """ + Generates compression arguments using cdvalueHelper.SZ3. + + Parameters: + - cmpr_algo (str): The compression algorithm. + - bound (float): The absolute error bound. + + Returns: + - tuple: Compression filter and options. + """ + try: + config = SZ3(cmpr_algo, absolute=bound) + return (32024, tuple(config.cd_values)) + except Exception as e: + print(f"Error generating compression arguments: {e}") + sys.exit(1) + + +def read_raw_data(raw_file_path, shape, dtype): + """ + Reads raw data from a file and reshapes it. + + Parameters: + - raw_file_path (str): Path to the raw file. + - shape (tuple): Desired shape of the data. + - dtype (str): Data type, 'float32' or 'float64'. + + Returns: + - np.ndarray: Reshaped data array. + """ + try: + # Mapping from dtype string to numpy dtype + np_dtypes = { + 'float32': np.float32, + 'float64': np.float64, + 'int32': np.int32, + 'int64': np.int64, + # Add more as needed + } + np_dtype = np_dtypes.get(dtype, np.float32) # Default to float32 + data = np.fromfile(raw_file_path, dtype=np_dtype) + expected_size = np.prod(shape) + if data.size != expected_size: + raise ValueError(f"Data size {data.size} does not match expected shape {shape}") + data = data.reshape(shape) + print(f"Successfully read and reshaped raw data from '{raw_file_path}' with shape {shape}") + return data + except Exception as e: + print(f"Error reading raw data: {e}") + sys.exit(1) + + +def biomd_refuses(cmpr_algo, chunk): + """ALGO_BIOMD takes chunks of coordinates, a last dimension of 3 after dimensions of 1 are dropped.""" + dims = [d for d in chunk if d > 1] + return cmpr_algo == 'ALGO_BIOMD' and (len(dims) > 3 or not dims or dims[-1] != 3) + + +def write_hdf5(data, h5_file_path, dataset_name='test', compression=None, compression_opts=None, chunks=None, + refusable=False): + """ + Writes data to an HDF5 file. + + Parameters: + - data (np.ndarray): Data to write. + - h5_file_path (str): Path to the HDF5 file. + - dataset_name (str): Name of the dataset within the HDF5 file. + - compression (str): Compression filter to use. + - compression_opts (tuple): Compression options. + - chunks (tuple or None): Chunk shape for the dataset. + - refusable (bool): raise when the filter refuses the dataset, instead of exiting. + """ + try: + with h5py.File(h5_file_path, 'w') as f: + dset = f.create_dataset(dataset_name, data=data, compression=compression, compression_opts=compression_opts, + chunks=chunks) + print(f"Actual chunk size for dataset '{dataset_name}': {dset.chunks}") + print(f"Data written to HDF5 file '{h5_file_path}' with dataset name '{dataset_name}'") + except Exception as e: + if refusable: + raise + print(f"Error writing HDF5 file: {e}") + sys.exit(1) + + +def compare_hdf5(original_h5, decompressed_h5, dataset_name='test'): + """ + Compares two HDF5 files and prints the maximum error between the datasets. + + Parameters: + - original_h5 (str): Path to the original HDF5 file. + - decompressed_h5 (str): Path to the decompressed HDF5 file. + - dataset_name (str): Name of the dataset to compare. + + Returns: + - float: The maximum error found between the datasets. + """ + try: + with h5py.File(original_h5, 'r') as f1, h5py.File(decompressed_h5, 'r') as f2: + data1 = f1[dataset_name][:] + data2 = f2[dataset_name][:] + if data1.shape != data2.shape: + raise ValueError("Shape mismatch between original and decompressed data.") + max_error = np.max(np.abs(data1 - data2)) + max_error_location = np.unravel_index(np.argmax(np.abs(data1 - data2)), data1.shape) + print(f"Max error between original and decompressed data: {max_error}") + print(f"Location of max error: {max_error_location}") + return max_error + except Exception as e: + print(f"Error comparing HDF5 files: {e}") + sys.exit(1) + + +def parse_shape(args): + """ + Parses shape dimensions from command-line arguments. + + Parameters: + - args (list): List of shape dimensions as strings. + + Returns: + - tuple: Parsed shape as a tuple of integers. + """ + try: + shape = tuple(int(dim) for dim in args) + return shape + except ValueError: + print("Error: All shape dimensions must be integers.") + sys.exit(1) + except Exception as e: + print(f"Unexpected error parsing shape: {e}") + sys.exit(1) + + +def main(): + if len(sys.argv) < 7: + print( + "Usage: python test_h5_filter.py [ ... ]") + sys.exit(1) + + h5_plugin_path = sys.argv[1] + cmpr_algo = sys.argv[2] + bound = float(sys.argv[3]) + raw_file = sys.argv[4] + dtype = sys.argv[5] + shape_args = sys.argv[6:] + + h5_dataset_name = 'testdata_compressed' + output_dir = tempfile.mkdtemp(prefix='output_hdf5_') + # output_dir='/var/tmp/output_hdf5_' + # os.makedirs(output_dir, exist_ok=True) + + shape = parse_shape(shape_args) + + base_name = os.path.splitext(os.path.basename(raw_file))[0] + original_h5 = os.path.join(output_dir, f"{base_name}_original.h5") + + data = read_raw_data(raw_file, shape, dtype) + + write_hdf5(data, original_h5, h5_dataset_name) + + compression, compression_opts = get_compression_args(cmpr_algo, bound) + + all_pass = True + # 'small' keeps a chunk well under the compressed-buffer bound, which the filter has to + # size from SZ_compress_size_bound rather than from the raw chunk size. A chunk that small + # over a whole field would mean tens of millions of chunks, whose HDF5 index alone exhausts + # memory, so the mode runs on a leading slice that still covers many chunks. + max_small_chunks = 4096 + small_chunk = tuple(min(d, 8) for d in shape) + for chunk in ['full', 'auto', 'small']: + print(f"Testing {raw_file} with algo = {cmpr_algo} AbsErrorBound = {bound} Chunk = {chunk}") + + compressed_h5 = os.path.join(output_dir, f"{base_name}_compressed_{chunk}.h5") + decompressed_h5 = os.path.join(output_dir, f"{base_name}_decompressed_{chunk}.h5") + + payload, reference_h5 = data, original_h5 + if chunk == 'small': + chunks_per_row = 1 + for axis in range(1, len(shape)): + chunks_per_row *= -(-shape[axis] // small_chunk[axis]) + rows = max(1, max_small_chunks // chunks_per_row) * small_chunk[0] + if rows < shape[0]: + payload = data[:rows] + reference_h5 = os.path.join(output_dir, f"{base_name}_original_{chunk}.h5") + write_hdf5(payload, reference_h5, h5_dataset_name) + print(f" restricted to the leading {rows} of {shape[0]} to bound the chunk count") + + # h5py chooses the chunk shape when chunks is not set ('auto') + chunks = None if chunk == 'auto' else payload.shape if chunk == 'full' else small_chunk + try: + write_hdf5(payload, compressed_h5, h5_dataset_name, compression=compression, + compression_opts=compression_opts, chunks=chunks, refusable=cmpr_algo == 'ALGO_BIOMD') + except Exception as e: + # ALGO_BIOMD refuses a chunk shape that is not coordinates when the dataset is created, naming the shape + named = re.search(r'these are \(([\d, ]+)\)', str(e)) + refused = named is not None and biomd_refuses(cmpr_algo, [int(d) for d in named.group(1).split(',')]) + result = "PASS" if refused and (chunks is None or biomd_refuses(cmpr_algo, chunks)) else "FAIL" + print(f" {e}") + print(f"Test Result for AbsErrorBound = {bound} Chunk = {chunk}: {result} (refused)") + all_pass = all_pass and result == "PASS" + continue + if chunks is not None and biomd_refuses(cmpr_algo, chunks): + print(f"Test Result for AbsErrorBound = {bound} Chunk = {chunk}: FAIL (chunks {chunks} were not refused)") + all_pass = False + continue + + with h5py.File(compressed_h5, 'r') as f_in, h5py.File(decompressed_h5, 'w') as f_out: + f_out.create_dataset(h5_dataset_name, data=f_in[h5_dataset_name][:]) + + max_error = compare_hdf5(reference_h5, decompressed_h5, h5_dataset_name) + + if max_error <= bound * 1.2: + result = "PASS" + else: + result = "FAIL" + + # Collected by the driver. The filter is the path a simulation writes through, so its + # ratio is the one that describes what a user's files cost. + print(f"METRICS chunk={chunk} bytes={os.path.getsize(compressed_h5)} " + f"ratio={os.path.getsize(reference_h5) / os.path.getsize(compressed_h5):.6f} " + f"max_error={max_error:.6g}") + print(f"Test Result for AbsErrorBound = {bound} Chunk = {chunk}: {result}") + + if result == "FAIL": + all_pass = False + + rmtree(output_dir) + + if not all_pass: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/test/integration/test_sz3_executable.py b/tools/test/integration/test_sz3_executable.py new file mode 100644 index 00000000..b3c0c5db --- /dev/null +++ b/tools/test/integration/test_sz3_executable.py @@ -0,0 +1,200 @@ +import numpy as np +import subprocess +import os +import sys +import time +from shutil import rmtree +import tempfile + + +def create_sz3_config(algo, path): + """ + Creates the sz3.config file. + """ + with open(os.path.join(path, "sz3.config"), "w") as f: + f.write(f"[GlobalSettings]\n") + f.write(f"CmprAlgo = {algo}\n") + + +def biomd_refuses(cmpr_algo, shape): + """ALGO_BIOMD takes coordinates, a last dimension of 3 after dimensions of 1 are dropped, and refuses the rest.""" + dims = [d for d in shape if d > 1] + return cmpr_algo == 'ALGO_BIOMD' and (len(dims) > 3 or not dims or dims[-1] != 3) + + +def run_sz3_compress(sz3_executable, input_file, output_file, bound, dims, cwd, dtype_flag): + """ + Runs the sz3 executable for compression. + """ + try: + cmd = [sz3_executable, dtype_flag, '-i', input_file, '-z', os.path.basename(output_file), '-c', 'sz3.config', + '-M', + 'ABS', str(bound), f'-{len(dims)}'] + dims + print(f"Running sz3 compression with command: {' '.join(cmd)}") + started = time.monotonic() + subprocess.run(cmd, check=True, cwd=cwd) + elapsed = time.monotonic() - started + print(f"sz3 compression completed. Output file: '{output_file}'") + return elapsed + except subprocess.CalledProcessError as e: + print(f"Error running sz3 compression: {e}") + print(f"stderr: {e.stderr}") + sys.exit(1) + + +def run_sz3_decompress(sz3_executable, compressed_file, decompressed_file, dims, cwd, dtype_flag): + """ + Runs the sz3 executable for decompression. + """ + try: + cmd = [sz3_executable, dtype_flag, '-s', os.path.basename(compressed_file), '-o', + os.path.basename(decompressed_file), f'-{len(dims)}'] + dims + print(f"Running sz3 decompression with command: {' '.join(cmd)}") + started = time.monotonic() + subprocess.run(cmd, check=True, cwd=cwd) + elapsed = time.monotonic() - started + print(f"sz3 decompression completed. Output file: '{decompressed_file}'") + return elapsed + except subprocess.CalledProcessError as e: + print(f"Error running sz3 decompression: {e}") + print(f"stderr: {e.stderr}") + sys.exit(1) + + +def read_raw_data(raw_file_path, shape, dtype): + """ + Reads raw data from a file and reshapes it. + """ + try: + # Mapping from dtype string to numpy dtype + np_dtypes = { + 'float32': np.float32, + 'float64': np.float64, + 'int32': np.int32, + 'int64': np.int64, + # Add more as needed + } + np_dtype = np_dtypes.get(dtype, np.float32) # Default to float32 + data = np.fromfile(raw_file_path, dtype=np_dtype) + expected_size = np.prod(shape) + if data.size != expected_size: + raise ValueError(f"Data size {data.size} does not match expected shape {shape}") + data = data.reshape(shape) + print(f"Successfully read and reshaped raw data from '{raw_file_path}' with shape {shape}") + return data + except Exception as e: + print(f"Error reading raw data: {e}") + sys.exit(1) + + +def compare_data(original_data, decompressed_data): + """ + Compares two numpy arrays and prints the maximum error. + """ + max_error = np.max(np.abs(original_data - decompressed_data)) + print(f"Max error between original and decompressed data: {max_error}") + return max_error + + +def parse_shape(args): + """ + Parses shape dimensions from command-line arguments. + """ + try: + shape = tuple(int(dim) for dim in args) + return shape + except ValueError: + print("Error: All shape dimensions must be integers.") + sys.exit(1) + except Exception as e: + print(f"Unexpected error parsing shape: {e}") + sys.exit(1) + + +def main(): + if len(sys.argv) < 7: + print( + "Usage: python test_sz3_executable.py [ ... ]") + sys.exit(1) + + sz3_exe_folder = sys.argv[1] + cmpr_algo = sys.argv[2] + bound = float(sys.argv[3]) + raw_file = sys.argv[4] + dtype = sys.argv[5] + shape_args = sys.argv[6:] + + # Mapping from dtype string to SZ3 flag + dtype_flags = { + 'float32': '-f', + 'float64': '-d', + 'int32': '-i', + 'int64': '-l', + # Add more as needed + } + dtype_flag = dtype_flags.get(dtype, '-f') # Default to -f if unknown + + sz3_executable = os.path.join(sz3_exe_folder, "sz3") if os.path.isdir(sz3_exe_folder) else sz3_exe_folder + + output_dir = tempfile.mkdtemp(prefix='output_sz3_') + + shape = parse_shape(shape_args) + + base_name = os.path.splitext(os.path.basename(raw_file))[0] + compressed_file = os.path.join(output_dir, f"{base_name}_compressed.sz3") + decompressed_file = os.path.join(output_dir, f"{base_name}_decompressed.dat") + + create_sz3_config(cmpr_algo, output_dir) + + if biomd_refuses(cmpr_algo, shape): + cmd = [sz3_executable, dtype_flag, '-i', raw_file, '-z', compressed_file, '-c', 'sz3.config', '-M', 'ABS', + str(bound), f'-{len(shape)}'] + [str(d) for d in shape[::-1]] + run = subprocess.run(cmd, cwd=output_dir, capture_output=True, text=True) + refused = run.returncode != 0 and 'ALGO_BIOMD' in run.stdout + run.stderr + print((run.stdout + run.stderr).strip()) + result = "PASS" if refused else "FAIL" + print(f"Test Result for AbsErrorBound = {bound}: {result} (data that is not coordinates is refused)") + rmtree(output_dir) + sys.exit(0 if refused else 1) + + original_wd = os.getcwd() + os.chdir(output_dir) + + compress_seconds = run_sz3_compress(sz3_executable, raw_file, compressed_file, bound, + [str(d) for d in shape[::-1]], output_dir, dtype_flag) + + decompress_seconds = run_sz3_decompress(sz3_executable, compressed_file, decompressed_file, + [str(d) for d in shape[::-1]], output_dir, dtype_flag) + + ratio = os.path.getsize(raw_file) / os.path.getsize(compressed_file) + + os.chdir(original_wd) + + original_data = read_raw_data(raw_file, shape, dtype) + decompressed_data = read_raw_data(decompressed_file, shape, dtype) + + max_error = compare_data(original_data, decompressed_data) + + if max_error > bound * 1.2: + result = "FAIL" + elif ratio < 1.0: + # No ratio is guaranteed, but producing a file larger than the input is a defect. + print(f"Compressed output is larger than the input: ratio {ratio:.3f}") + result = "FAIL" + else: + result = "PASS" + + # Collected by the driver so a ratio or a timing can be compared across runs. + print(f"METRICS bytes={os.path.getsize(compressed_file)} ratio={ratio:.6f} " + f"compress_s={compress_seconds:.3f} decompress_s={decompress_seconds:.3f} " + f"max_error={max_error:.6g}") + print(f"Test Result for AbsErrorBound = {bound}: {result}") + + rmtree(output_dir) + + if result == "FAIL": + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/test/integration/update_compression_baseline.py b/tools/test/integration/update_compression_baseline.py new file mode 100644 index 00000000..76afe2a3 --- /dev/null +++ b/tools/test/integration/update_compression_baseline.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Merge the metrics CSVs a workflow run produced into the compression baseline. + +The integration jobs each write an integration_metrics.csv and keep it as an artifact. Download +those, point this at them, and commit the result. Do that when a change is meant to move the +compressed size; the driver fails a case whose size has drifted more than the tolerance it +carries, which is what makes an unintended move visible. + +Usage: + python3 update_compression_baseline.py [ ...] +""" + +import argparse +import csv +import json +import os + +BASELINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "compression_baseline.json") +NOTE = ("Compressed size in bytes per dataset/field|algo|error bound, from a run on master. " + "Regenerate with update_compression_baseline.py when a change is meant to move it.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("csv_files", nargs="+") + parser.add_argument("--baseline", default=BASELINE) + parser.add_argument("--prune", action="store_true", + help="drop entries the given CSVs do not mention") + args = parser.parse_args() + + try: + with open(args.baseline) as handle: + baseline = json.load(handle) + except FileNotFoundError: + baseline = {} + + seen, changed, added = set(), 0, 0 + for path in args.csv_files: + with open(path, newline="") as handle: + for row in csv.DictReader(handle): + if not row.get("bytes"): + continue + key = f"{row['dataset']}/{row['field']}|{row['algo']}|{row['error_bound']}" + value = int(row["bytes"]) + seen.add(key) + if key not in baseline: + added += 1 + elif baseline[key] != value: + print(f"{key}: {baseline[key]} -> {value}") + changed += 1 + baseline[key] = value + + if args.prune: + for key in [k for k in baseline if k not in seen and not k.startswith("_")]: + print(f"dropping {key}") + del baseline[key] + + baseline["_comment"] = NOTE + ordered = {"_comment": baseline.pop("_comment")} + ordered.update(dict(sorted(baseline.items()))) + with open(args.baseline, "w") as handle: + json.dump(ordered, handle, indent=2) + handle.write("\n") + print(f"{len(ordered) - 1} entries in {args.baseline}: {added} added, {changed} changed") + + +if __name__ == "__main__": + main() diff --git a/tools/test/modules/test_biomd.cpp b/tools/test/modules/test_biomd.cpp new file mode 100644 index 00000000..d6228eb4 --- /dev/null +++ b/tools/test/modules/test_biomd.cpp @@ -0,0 +1,228 @@ +// ALGO_BIOMD, through the public API a caller reaches it by. +// +// The integration suite runs it on the SDRBench fields, all of which are 1D or 2D, so the +// trajectory layout it was written for -- dims {frames, atoms, xyz} -- is +// only ever reached through 3D climate fields. Those never have a frame whose values are all +// equal, so the fill-frame path that skips trailing frames and restores them on decompression +// does not run there at all. A trajectory buffer that holds fewer frames than it has room for +// does, which is the case a simulation writing through the HDF5 filter produces. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" +#include "gtest/gtest.h" + +namespace { + +/// A water-box trajectory: atoms placed on a jittered lattice, then walking a little each frame. +std::vector make_trajectory(size_t frames, size_t atoms, uint32_t seed = 7) { + std::mt19937 rng(seed); + std::uniform_real_distribution jitter(-0.02f, 0.02f); + std::vector positions(atoms * 3); + const auto side = static_cast(std::cbrt(static_cast(atoms)) + 1); + for (size_t a = 0; a < atoms; a++) { + positions[a * 3 + 0] = static_cast(a % side) * 0.31f; + positions[a * 3 + 1] = static_cast((a / side) % side) * 0.31f; + positions[a * 3 + 2] = static_cast(a / (side * side)) * 0.31f; + } + + std::vector traj(frames * atoms * 3); + for (size_t f = 0; f < frames; f++) { + for (size_t i = 0; i < atoms * 3; i++) { + positions[i] += jitter(rng); + traj[f * atoms * 3 + i] = positions[i]; + } + } + return traj; +} + +/// Compress and decompress through the public API, returning the compressed bytes. +std::vector round_trip(double eb, const std::vector &dims, const std::vector &input, + std::vector &output) { + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + + std::vector compressed(SZ3::SZ_compress_size_bound(conf)); + const size_t size = SZ_compress(conf, input.data(), compressed.data(), compressed.size()); + compressed.resize(size); + + output.assign(input.size(), 0.0f); + float *out = output.data(); + SZ3::Config read_conf; + SZ_decompress(read_conf, compressed.data(), compressed.size(), out); + return compressed; +} + +float max_abs_error(const std::vector &a, const std::vector &b) { + float worst = 0; + for (size_t i = 0; i < a.size(); i++) { + worst = std::max(worst, std::fabs(a[i] - b[i])); + } + return worst; +} + +} // namespace + +/// dims {frames, atoms, xyz} is the layout a trajectory arrives in. +TEST(SZ3_BioMD, TrajectoryRoundTripHonoursTheBound) { + const std::vector dims = {21, 2652, 3}; + const auto input = make_trajectory(dims[0], dims[1]); + + for (double eb : {1e-1, 1e-2, 1e-3, 1e-4}) { + std::vector output; + const auto compressed = round_trip(eb, dims, input, output); + ASSERT_EQ(output.size(), input.size()) << " eb=" << eb; + EXPECT_LE(max_abs_error(input, output), eb) << " eb=" << eb; + EXPECT_LT(compressed.size(), input.size() * sizeof(float)) << " eb=" << eb << " did not compress"; + } +} + +/// Trailing frames that hold one value are skipped on compression and written back on decompression. +TEST(SZ3_BioMD, TrailingFilledFramesAreRestored) { + const std::vector dims = {16, 500, 3}; + const size_t frame = dims[1] * dims[2]; + const size_t written_frames = 9; + const float fill = 0.12345f; + + auto input = make_trajectory(dims[0], dims[1]); + for (size_t f = written_frames; f < dims[0]; f++) { + std::fill(input.begin() + f * frame, input.begin() + (f + 1) * frame, fill); + } + + std::vector output; + const auto compressed = round_trip(1e-3, dims, input, output); + ASSERT_EQ(output.size(), input.size()); + EXPECT_LE(max_abs_error(input, output), 1e-3); + // The filled tail is written back verbatim, not quantized. + for (size_t i = written_frames * frame; i < input.size(); i++) { + ASSERT_EQ(output[i], fill) << " at element " << i; + } + // Skipping the filled tail has to cost less than quantizing it, or nothing here would + // notice the skip being removed. + std::vector without_fill_output; + const auto without_fill = round_trip(1e-3, dims, make_trajectory(dims[0], dims[1]), without_fill_output); + EXPECT_LT(compressed.size(), without_fill.size()); +} + +/// Everything after the first frame is fill, which is the largest the skipped range can be. +TEST(SZ3_BioMD, EveryFrameAfterTheFirstIsFill) { + const std::vector dims = {8, 300, 3}; + const size_t frame = dims[1] * dims[2]; + const float fill = -3.14159f; + + auto input = make_trajectory(dims[0], dims[1]); + std::fill(input.begin() + frame, input.end(), fill); + + std::vector output; + round_trip(1e-3, dims, input, output); + ASSERT_EQ(output.size(), input.size()); + EXPECT_LE(max_abs_error(input, output), 1e-3); + for (size_t i = frame; i < input.size(); i++) { + ASSERT_EQ(output[i], fill) << " at element " << i; + } +} + +/// The chunk shape a filter writes when it stores one frame at a time. Config::setDims drops a +/// dimension of 1, so this reaches the two-dimensional path rather than the trajectory one. +TEST(SZ3_BioMD, OneFramePerChunk) { + const std::vector dims = {1, 1024, 3}; + const auto input = make_trajectory(dims[0], dims[1]); + + std::vector output; + round_trip(1e-3, dims, input, output); + ASSERT_EQ(output.size(), input.size()); + EXPECT_LE(max_abs_error(input, output), 1e-3); +} + +/// The smallest trajectory that still reaches the multi-frame path. +TEST(SZ3_BioMD, TwoFrameTrajectory) { + const std::vector dims = {2, 1024, 3}; + const auto input = make_trajectory(dims[0], dims[1]); + + std::vector output; + round_trip(1e-3, dims, input, output); + ASSERT_EQ(output.size(), input.size()); + EXPECT_LE(max_abs_error(input, output), 1e-3); +} + +/// ALGO_BIOMD takes coordinates, {..., 3}, and throws on other shapes. +bool refused(const std::vector &dims) { return dims.back() != 3; } + +/// The single-frame paths, which 1D and 2D data take. +TEST(SZ3_BioMD, OneAndTwoDimensionalInput) { + const auto atoms = make_trajectory(1, 4096); + + for (const std::vector &dims : {std::vector{atoms.size()}, std::vector{4096, 3}}) { + std::vector output; + if (refused(dims)) { + EXPECT_THROW(round_trip(1e-3, dims, atoms, output), std::invalid_argument); + continue; + } + round_trip(1e-3, dims, atoms, output); + ASSERT_EQ(output.size(), atoms.size()) << " " << dims.size() << "D"; + EXPECT_LE(max_abs_error(atoms, output), 1e-3) << " " << dims.size() << "D"; + } +} + +/// Inputs shorter than two atoms are refused or round-trip within the bound. +TEST(SZ3_BioMD, InputsShorterThanTwoAtoms) { + for (size_t n : {size_t{1}, size_t{2}, size_t{3}, size_t{5}}) { + std::vector input(n); + for (size_t i = 0; i < n; i++) { + input[i] = 0.5f * static_cast(i) - 1.0f; + } + std::vector output; + if (refused({n})) { + EXPECT_THROW(round_trip(1e-3, {n}, input, output), std::invalid_argument); + continue; + } + round_trip(1e-3, {n}, input, output); + ASSERT_EQ(output.size(), n) << " n=" << n; + EXPECT_LE(max_abs_error(input, output), 1e-3) << " n=" << n; + } +} + +/// The same input has to produce the same file, or a trajectory cannot be checksummed. +TEST(SZ3_BioMD, CompressionIsDeterministic) { + // A decomposition member that one compress() path leaves unassigned reaches save() from + // whatever the memory held, and the paths differ by dimensionality, so all three are here. + const std::vector> shapes = {{12288}, {64, 64}, {5, 777, 3}}; + for (const auto &dims : shapes) { + const size_t atoms = dims.size() == 3 ? dims[1] : 4096; + const auto input = make_trajectory(dims.size() == 3 ? dims[0] : 1, atoms); + std::vector discard; + if (refused(dims)) continue; + const auto first = round_trip(1e-3, dims, input, discard); + const auto second = round_trip(1e-3, dims, input, discard); + ASSERT_EQ(first.size(), second.size()) << " " << dims.size() << "D"; + EXPECT_EQ(0, std::memcmp(first.data(), second.data(), first.size())) << " " << dims.size() << "D"; + } +} + +/// ALGO_BIOMDXTC was removed; asking for it, also with a bound that would go lossless, says to use ALGO_BIOMD. +TEST(SZ3_BioMD, RemovedXtcAlgorithmIsRefused) { + const auto input = make_trajectory(2, 100); + for (double eb : {1e-3, 0.0}) { + SZ3::Config conf(2, 100, 3); + conf.cmprAlgo = SZ3::ALGO_BIOMDXTC; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + size_t size = 0; + try { + delete[] SZ_compress(conf, input.data(), size); + ADD_FAILURE() << "ALGO_BIOMDXTC compressed at eb=" << eb; + } catch (const std::invalid_argument &e) { + EXPECT_NE(std::string(e.what()).find("ALGO_BIOMD"), std::string::npos) << e.what(); + } + } +} diff --git a/tools/test/modules/test_biomd_codec.cpp b/tools/test/modules/test_biomd_codec.cpp new file mode 100644 index 00000000..83e32604 --- /dev/null +++ b/tools/test/modules/test_biomd_codec.cpp @@ -0,0 +1,614 @@ +// ALGO_BIOMD on synthetic molecular-dynamics systems: rigid 3- and 4-site water, bonded chains and ions, as GROMACS +// lays them out (molecules in order, water in one block): the bound, the input it refuses, recompression, and corrupt +// or truncated streams, which are refused without reading or writing out of bounds. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" +#include "gtest/gtest.h" + +namespace { + +struct SystemSpec { + size_t waters = 800; + bool four_site = false; + size_t chains = 2, chain_len = 60; // heavy atoms per chain, each with one hydrogen + size_t ions = 10; + size_t frames = 1; + uint32_t seed = 1; +}; + +struct Vec { + double x, y, z; +}; +Vec operator+(Vec a, Vec b) { return {a.x + b.x, a.y + b.y, a.z + b.z}; } +Vec operator*(double s, Vec a) { return {s * a.x, s * a.y, s * a.z}; } +double dot(Vec a, Vec b) { return a.x * b.x + a.y * b.y + a.z * b.z; } +Vec unit(Vec a) { return (1.0 / std::sqrt(dot(a, a))) * a; } +Vec cross(Vec a, Vec b) { return {a.y * b.z - a.z * b.y, a.z * b.x - a.x * b.z, a.x * b.y - a.y * b.x}; } + +// frames x atoms x 3 floats, in nm +std::vector make_system(const SystemSpec &s, size_t *atoms) { + std::mt19937 rng(s.seed); + std::uniform_real_distribution U(-1, 1); + std::normal_distribution G(0, 1); + auto rvec = [&]() { return unit(Vec{G(rng), G(rng), G(rng)}); }; + const size_t per_water = s.four_site ? 4 : 3; + const size_t n = s.chains * s.chain_len * 2 + s.waters * per_water + s.ions; + *atoms = n; + const double box = std::cbrt(double(n) * 0.01); + const double r = 0.09572, theta = 104.52 * 3.14159265358979323846 / 180, a_vs = 0.128; // TIP4P geometry + const double bonds[3] = {0.1529, 0.1335, 0.1471}, bh = 0.109; + std::vector O(s.waters), u(s.waters), v(s.waters), chain_dir(s.chains * s.chain_len), ion(s.ions); + std::vector chain0(s.chains); + for (size_t w = 0; w < s.waters; w++) { + O[w] = {box * (U(rng) + 1) / 2, box * (U(rng) + 1) / 2, box * (U(rng) + 1) / 2}; + u[w] = rvec(); + v[w] = unit(cross(u[w], rvec())); + } + for (auto &d : chain_dir) d = rvec(); + for (auto &c : chain0) c = {box * (U(rng) + 1) / 2, box * (U(rng) + 1) / 2, box * (U(rng) + 1) / 2}; + for (auto &i : ion) i = {box * (U(rng) + 1) / 2, box * (U(rng) + 1) / 2, box * (U(rng) + 1) / 2}; + std::vector x(s.frames * n * 3); + for (size_t f = 0; f < s.frames; f++) { + float *X = &x[f * n * 3]; + size_t k = 0; + auto put = [&](Vec p) { + X[3 * k] = float(p.x); + X[3 * k + 1] = float(p.y); + X[3 * k + 2] = float(p.z); + k++; + }; + for (size_t c = 0; c < s.chains; c++) { + Vec p = chain0[c]; + for (size_t a = 0; a < s.chain_len; a++) { + const Vec d = chain_dir[c * s.chain_len + a]; + p = p + bonds[a % 3] * d; + put(p); + put(p + bh * unit(cross(d, rvec()))); + } + } + for (size_t w = 0; w < s.waters; w++) { + const Vec h1 = O[w] + r * u[w], h2 = O[w] + r * (std::cos(theta) * u[w] + std::sin(theta) * v[w]); + put(O[w]); + put(h1); + put(h2); + if (s.four_site) put(O[w] + a_vs * ((h1 + (-1.0) * O[w]) + (h2 + (-1.0) * O[w]))); + } + for (auto &p : ion) put(p); + // the next frame: molecules move and turn a little + for (size_t w = 0; w < s.waters; w++) { + O[w] = O[w] + 0.003 * rvec(); + u[w] = unit(u[w] + 0.05 * rvec()); + v[w] = unit(cross(u[w], cross(v[w], u[w]))); + } + for (auto &d : chain_dir) d = unit(d + 0.03 * rvec()); + for (auto &p : ion) p = p + 0.005 * rvec(); + } + return x; +} + +template +struct RoundTrip { + std::vector bytes; + std::vector out; + SZ3::ALGO algo; + double max_err; +}; + +template +RoundTrip round_trip(const std::vector &x, size_t frames, size_t atoms, double eb) { + SZ3::Config conf; + if (frames == 1) + conf = SZ3::Config(atoms, 3); + else + conf = SZ3::Config(frames, atoms, 3); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + size_t size = 0; + char *c = SZ_compress(conf, x.data(), size); + RoundTrip r; + r.bytes.assign(c, c + size); + delete[] c; + SZ3::Config dconf; + r.out.resize(x.size()); + T *o = r.out.data(); + SZ_decompress(dconf, r.bytes.data(), r.bytes.size(), o); + r.algo = static_cast(dconf.cmprAlgo); // the algorithm the stream holds + r.max_err = 0; + for (size_t i = 0; i < x.size(); i++) r.max_err = std::max(r.max_err, std::fabs(double(r.out[i]) - double(x[i]))); + return r; +} + +TEST(BioMD, WithinBoundAndBeatsFourBytesPerValue) { + for (bool four : {false, true}) + for (size_t frames : {size_t(1), size_t(5), size_t(20)}) + for (double eb : {5e-4, 1e-3, 1e-5, 0.1}) { + SystemSpec s; + s.four_site = four; + s.frames = frames; + size_t n; + const auto x = make_system(s, &n); + const auto r = round_trip(x, frames, n, eb); + SCOPED_TRACE(testing::Message() << "four_site=" << four << " frames=" << frames << " eb=" << eb); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD); + EXPECT_LE(r.max_err, eb); + if (eb == 5e-4) { + EXPECT_GT(double(x.size() * 4) / double(r.bytes.size()), 3.0); + } + } +} + +TEST(BioMD, LooseBoundsStayWithinTheBound) { + SystemSpec s; + s.frames = 20; + size_t n; + auto x = make_system(s, &n); + for (size_t t = 1; t < s.frames; t++) // frames that do not move + std::copy(x.begin(), x.begin() + 3 * n, x.begin() + t * 3 * n); + for (double eb : {0.02, 0.1}) { + const auto r = round_trip(x, s.frames, n, eb); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD); + EXPECT_LE(r.max_err, eb); + } +} + +TEST(BioMD, DoubleInput) { + SystemSpec s; + s.frames = 4; + size_t n; + const auto xf = make_system(s, &n); + const std::vector x(xf.begin(), xf.end()); + for (double eb : {5e-4, 1e-7}) { + const auto r = round_trip(x, s.frames, n, eb); + EXPECT_LE(r.max_err, eb) << "eb=" << eb; + } +} + +TEST(BioMD, EdgeCasesStayWithinBound) { + std::mt19937 rng(3); + std::uniform_real_distribution U(0.f, 5.f); + std::vector>> cases; + for (size_t n : {1, 2, 3, 4, 5, 7}) { + std::vector x(3 * n); + for (auto &v : x) v = U(rng); + cases.push_back({"tiny N=" + std::to_string(n), x}); + } + cases.push_back({"constant", std::vector(3000, 1.234f)}); + cases.push_back({"zeros", std::vector(3000, 0.f)}); + SystemSpec s; + size_t n; + const auto base = make_system(s, &n); + { + auto x = base; // waters split over a periodic boundary + for (size_t i = s.chains * s.chain_len * 2; i + 2 < n; i += 30) x[3 * (i + 1)] += 3.1f; + cases.push_back({"broken waters", x}); + } + { + auto x = base; // atoms in no particular order + std::vector perm(n); + std::iota(perm.begin(), perm.end(), 0); + std::shuffle(perm.begin(), perm.end(), rng); + for (size_t i = 0; i < n; i++) + for (int c = 0; c < 3; c++) x[3 * i + c] = base[3 * perm[i] + c]; + cases.push_back({"shuffled", x}); + } + { + auto x = base; + for (auto &v : x) v += 1600.f; // within the lattice of 5e-4 (2048 nm) + cases.push_back({"offset 1600 nm", x}); + } + { + auto x = base; + for (auto &v : x) v *= 10.f; // Angstrom + cases.push_back({"Angstrom", x}); + } + for (const auto &c : cases) { + const size_t atoms = c.second.size() / 3; + const auto r = round_trip(c.second, 1, atoms, 5e-4); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD) << c.first; + EXPECT_LE(r.max_err, 5e-4) << c.first; + } +} + +// Input BIOMD does not code: shapes other than {frames, atoms, 3} go to LORENZO_REG, within the bound; coordinates +// beyond the lattice the bound allows, NaN or Inf outside trailing fill, and bounds that are not positive and finite, +// to lossless compression. The input stays as it was. +template +std::pair> compress_decompress(const std::vector &x, const std::vector &dims, + double eb) { + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + size_t size = 0; + const std::vector before = x; + std::unique_ptr cmp(SZ_compress(conf, x.data(), size)); + EXPECT_EQ(memcmp(before.data(), x.data(), x.size() * sizeof(T)), 0) << "the input changed"; + SZ3::Config dec; + std::unique_ptr y(SZ_decompress(dec, cmp.get(), size)); + return {static_cast(dec.cmprAlgo), std::vector(y.get(), y.get() + x.size())}; +} + +template +void expect_within(const std::vector &x, const std::vector &y, double eb) { + for (size_t i = 0; i < x.size(); i++) ASSERT_LE(std::fabs(double(y[i]) - double(x[i])), eb) << "at " << i; +} + +TEST(BioMD, FallsBackOnWhatItDoesNotCode) { + SystemSpec s; + size_t n; + const auto x = make_system(s, &n); + EXPECT_EQ(compress_decompress(x, {n, 3}, 5e-4).first, SZ3::ALGO_BIOMD); + for (const std::vector &dims : {std::vector{x.size()}, std::vector{3, n}, + std::vector{2, 5, n / 10, 3}}) { // n = 2650 + EXPECT_THROW(compress_decompress(x, dims, 5e-4), std::invalid_argument); + } + { + auto y = x; + for (auto &v : y) v += 3e3f; // beyond the lattice of 5e-4 (2048 nm) + const auto r = compress_decompress(y, {n, 3}, 5e-4); + EXPECT_EQ(r.first, SZ3::ALGO_LOSSLESS); + EXPECT_EQ(memcmp(r.second.data(), y.data(), y.size() * sizeof(float)), 0); + } + for (float bad : {std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity()}) { + auto y = x; + y[100] = bad; + const auto r = compress_decompress(y, {n, 3}, 5e-4); + EXPECT_EQ(r.first, SZ3::ALGO_LOSSLESS); + EXPECT_EQ(memcmp(r.second.data(), y.data(), y.size() * sizeof(float)), 0); + } + // bounds that are not positive and finite, ABS ones and REL over data with Inf, go to lossless compression + for (double eb : {double(INFINITY), double(NAN), -1e-3}) { + const auto r = compress_decompress(x, {n, 3}, eb); + EXPECT_EQ(r.first, SZ3::ALGO_LOSSLESS) << eb; + EXPECT_EQ(memcmp(r.second.data(), x.data(), x.size() * sizeof(float)), 0) << eb; + } + { + auto y = x; + y[100] = std::numeric_limits::infinity(); + SZ3::Config conf(n, 3); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.errorBoundMode = SZ3::EB_REL; + conf.relErrorBound = 1e-4; + size_t size = 0; + std::unique_ptr cmp(SZ_compress(conf, y.data(), size)); + SZ3::Config dec; + std::unique_ptr z(SZ_decompress(dec, cmp.get(), size)); + EXPECT_EQ(memcmp(z.get(), y.data(), y.size() * sizeof(float)), 0); + } + // a bound so large that the lattice would pass the largest value of the type + for (const auto &c : {std::make_pair(3e38f, 1e38), std::make_pair(1.0f, 1e30)}) { + const std::vector y = {c.first, 0, 0, 0, c.first, 0}; + const auto r = compress_decompress(y, {2, 3}, c.second); + for (float v : r.second) EXPECT_TRUE(std::isfinite(v)); + expect_within(y, r.second, c.second); + } + const std::vector z = {1.5e308, 0, 0, 0, 1.5e308, 0}; + const auto r = compress_decompress(z, {2, 3}, 5e307); + for (double v : r.second) EXPECT_TRUE(std::isfinite(v)); + expect_within(z, r.second, 5e307); +} + +// A chunk BIOMD coded, decompressed and compressed again with a NaN in a frame appended to it (HDF5 rewriting a +// chunk): the frames BIOMD decoded stay within the bound of the original data. +TEST(BioMD, NaNAppendedToACodedChunkKeepsItsFramesWithinTheBound) { + SystemSpec s; + s.frames = 10; + size_t n; + const auto x = make_system(s, &n); + const double eb = 5e-4; + const auto first = compress_decompress(x, {s.frames, n, 3}, eb); + ASSERT_EQ(first.first, SZ3::ALGO_BIOMD); + auto y = first.second; + y[9 * n * 3 + 100] = std::numeric_limits::quiet_NaN(); + const auto second = compress_decompress(y, {s.frames, n, 3}, eb); + EXPECT_EQ(second.first, SZ3::ALGO_LOSSLESS); + for (size_t i = 0; i < 9 * n * 3; i++) ASSERT_LE(std::fabs(double(second.second[i]) - double(x[i])), eb) << i; +} + +// Water detection probes every atoms / 300th atom: with a probe every 8 atoms, 4-site waters are met at the same place +// in each, which leading atoms move through O, H1, H2 and M. The waters must be found from each. +TEST(BioMD, FourSiteWaterIsFoundWhereverTheProbesLand) { + SystemSpec s; + s.four_site = true; + s.waters = 600; // 2650 atoms, and up to 3 more: a probe every 8 atoms + size_t n; + const auto base = make_system(s, &n); + std::vector ratios; + for (size_t k = 0; k < 4; k++) { + std::vector x; + for (size_t a = 0; a < k; a++) x.insert(x.end(), {0.5f + float(a), 0.5f, 0.5f}); + x.insert(x.end(), base.begin(), base.end()); + const auto r = round_trip(x, 1, n + k, 5e-4); + EXPECT_LE(r.max_err, 5e-4); + ratios.push_back(double(x.size() * sizeof(float)) / double(r.bytes.size())); + } + EXPECT_GT(*std::min_element(ratios.begin(), ratios.end()) / *std::max_element(ratios.begin(), ratios.end()), 0.95); +} + +// A frame of beads with no bond in range (coarse-grained water) is stored as raw lattice points in its box: within +// the bound, in about the bits of its box, and the same values when compressed again. +TEST(BioMD, BeadsWithoutBondsAreStoredAsLatticePoints) { + std::mt19937 rng(5); + std::uniform_real_distribution U(0.f, 10.f); + std::vector x(3 * 3000); + for (auto &v : x) v = U(rng); + for (double eb : {5e-3, 5e-4}) { + const auto r = round_trip(x, 1, 3000, eb); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD); + EXPECT_LE(r.max_err, eb); + const double bits = 3 * std::ceil(std::log2(10.0 / (2 * eb))); // a 10 nm box at about 2 eb per point + EXPECT_LT(double(r.bytes.size()) * 8, 1.05 * bits * 3000 + 2048) << eb; + const auto again = round_trip(r.out, 1, 3000, eb); + EXPECT_EQ(again.out, r.out) << eb; + } +} + +// Coordinates within half a step below the lattice span B: decompressed onto lattice points that may round past +// their originals, they must still be BIOMD's and compress to the same values, round after round. +TEST(BioMD, CoordinatesJustBelowTheSpanCompressTheSameAgain) { + SystemSpec s; + size_t n; + auto x = make_system(s, &n); + const double eb = 5e-4, span = 2048; // B for float at 5e-4: the power of two above 2^21 eb + float hi = 0; + for (float v : x) hi = std::max(hi, v); + for (double gap : {0.0004, 0.0006, 0.0011}) { + auto y = x; + for (auto &v : y) v = float(v + (span - gap - hi)); + auto r = round_trip(y, 1, n, eb); + ASSERT_EQ(r.algo, SZ3::ALGO_BIOMD) << gap; + for (int round = 0; round < 3; round++) { + const auto again = round_trip(r.out, 1, n, eb); + EXPECT_EQ(again.algo, SZ3::ALGO_BIOMD) << gap; + EXPECT_EQ(again.out, r.out) << gap << " round " << round; + r = again; + } + for (size_t i = 0; i < y.size(); i++) ASSERT_LE(std::fabs(double(r.out[i]) - double(y[i])), eb) << gap; + } +} + +// Every float from 3 steps below the lattice span B to 2 above, alone at the end of a frame, at bounds whose step falls +// in different places of a float ulp: a value whose rounded lattice point reaches B is stored losslessly, as one on a +// lattice point at B would round to a float of the binade above, past the bound. +TEST(BioMD, ValuesAroundTheSpanStayWithinTheBound) { + SystemSpec s; + size_t n; + const auto x = make_system(s, &n); + for (double eb : {5e-4, std::ldexp(1.0, -11), 3e-5, 7.3e-4}) { + int exponent; + std::frexp(std::ldexp(eb, SZ3::biomd::LATTICE_SPAN_BITS), &exponent); + const float span = float(std::ldexp(1.0, exponent)); + for (float sign : {1.f, -1.f}) + for (float v = span - float(6 * eb); v < span + float(4 * eb); v = std::nextafter(v, 2 * span)) { + auto y = x; + y.back() = sign * v; + const auto r = round_trip(y, 1, n, eb); + ASSERT_LE(r.max_err, eb) << eb << " " << sign * v; + } + } +} + +// A chunk rewritten as frames are appended and cut back (a crashed run resumed): its last frame goes past the lattice +// span and back, so the chunk goes from BIOMD to lossless storage and back, again and again. The frames before stay +// within the bound of the original data. +TEST(BioMD, ChunkRewrittenAcrossTheSpanStaysWithinTheBound) { + SystemSpec s; + s.frames = 4; + size_t n; + const auto x = make_system(s, &n); + const double eb = 5e-4; + auto y = compress_decompress(x, {s.frames, n, 3}, eb).second; + for (int round = 0; round < 3; round++) { + auto out = y; + for (size_t i = 3 * n * 3; i < 4 * n * 3; i++) out[i] = x[i] + 2400.f; // past B = 2048 nm + const auto past = compress_decompress(out, {s.frames, n, 3}, eb); + EXPECT_EQ(past.first, SZ3::ALGO_LOSSLESS); + y = past.second; + std::copy(x.begin() + 3 * n * 3, x.end(), y.begin() + 3 * n * 3); // the frame written again, within B + const auto back = compress_decompress(y, {s.frames, n, 3}, eb); + EXPECT_EQ(back.first, SZ3::ALGO_BIOMD); + y = back.second; + for (size_t i = 0; i < 3 * n * 3; i++) + ASSERT_LE(std::fabs(double(y[i]) - double(x[i])), eb) << round << " " << i; + } +} + +// Config drops dimensions of 1: one atom arrives as {frames, 3}, one atom of one frame as {3}. +TEST(BioMD, OneAtom) { + for (size_t frames : {size_t(1), size_t(5)}) { + std::vector x(frames * 3); + for (size_t i = 0; i < x.size(); i++) x[i] = 1.0f + 0.01f * float(i); + SZ3::Config conf(frames, 1, 3); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 5e-4; + size_t size = 0; + char *c = SZ_compress(conf, x.data(), size); + std::vector y(x.size()); + float *o = y.data(); + SZ3::Config dconf; + SZ_decompress(dconf, c, size, o); + delete[] c; + EXPECT_EQ(dconf.cmprAlgo, SZ3::ALGO_BIOMD); + for (size_t i = 0; i < x.size(); i++) EXPECT_LE(std::fabs(y[i] - x[i]), 5e-4) << frames << " frames"; + } +} + +TEST(BioMD, TrailingFillFramesComeBackExactly) { + SystemSpec s; + s.frames = 8; + size_t n; + auto x = make_system(s, &n); + std::fill(x.begin() + 5 * n * 3, x.end(), 0.f); // a chunk with 5 frames written + const auto r = round_trip(x, s.frames, n, 5e-4); + EXPECT_LE(r.max_err, 5e-4); + for (size_t i = 5 * n * 3; i < x.size(); i++) ASSERT_EQ(r.out[i], 0.f); +} + +// GROMACS rewrites a chunk after restarting from a checkpoint: decompress, append, compress again. +TEST(BioMD, RecompressionChangesNothing) { + for (size_t frames : {size_t(1), size_t(6)}) + for (double eb : {5e-4, 1e-3}) { + SystemSpec s; + s.frames = frames; + size_t n; + const auto x = make_system(s, &n); + const auto a = round_trip(x, frames, n, eb); + const auto b = round_trip(a.out, frames, n, eb); + EXPECT_EQ(0, memcmp(a.out.data(), b.out.data(), a.out.size() * sizeof(float))) << frames << " " << eb; + EXPECT_LE(b.max_err, eb); + } +} + +// The lattice depends on the bound alone: a decoded value that crosses a power of two, or a frame appended to a chunk +// that raises its largest |x|, lands on the same lattice points when compressed again. +TEST(BioMD, RecompressionAcrossAPowerOfTwoStaysWithinTheBound) { + const double eb = 5e-4; + SystemSpec s; + s.frames = 4; + size_t n; + auto x = make_system(s, &n); + size_t k = 0; + for (size_t i = 0; i < x.size(); i++) + if (std::fabs(x[i]) > std::fabs(x[k])) k = i; + const double p2 = std::exp2(std::ceil(std::log2(std::fabs(double(x[k]))))); + x[k] = float(std::copysign(p2 - 1e-4, double(x[k]))); + auto cur = x; + for (int round = 0; round < 3; round++) { + cur = round_trip(cur, s.frames, n, eb).out; + for (size_t i = 0; i < x.size(); i++) ASSERT_LE(std::fabs(double(cur[i]) - double(x[i])), eb) << round; + } + // two frames and two of fill, then a third frame whose largest |x| is past the power of two + std::vector chunk(x.begin(), x.begin() + 2 * n * 3); + chunk.resize(4 * n * 3, 0.f); + auto dec = round_trip(chunk, 4, n, eb).out; + for (size_t i = 0; i < n * 3; i++) dec[2 * n * 3 + i] = x[2 * n * 3 + i]; + dec[2 * n * 3] = float(p2 + 0.5); + const auto again = round_trip(dec, 4, n, eb).out; + for (size_t i = 0; i < 2 * n * 3; i++) ASSERT_LE(std::fabs(double(again[i]) - double(x[i])), eb) << i; +} + +TEST(BioMD, CorruptStreamsAreRefusedOrDecodeInBounds) { + SystemSpec s; + s.frames = 3; + s.four_site = true; + size_t n; + const auto x = make_system(s, &n); + const auto r = round_trip(x, s.frames, n, 5e-4); + // SZ3's stream: 16-byte header (magic, version, payload size), the payload, then the config + uint64_t payload = 0; + memcpy(&payload, r.bytes.data() + 8, 8); + ASSERT_LT(16 + payload, r.bytes.size()); + std::vector out(x.size()); + std::mt19937 rng(9); + auto attempt = [&](std::vector b) { + try { + SZ3::Config dconf; + float *o = out.data(); + SZ_decompress(dconf, b.data(), b.size(), o); + } catch (std::exception &) { + } + }; + for (size_t cut = 16; cut < 16 + payload; cut += std::max(1, payload / 300)) { // payload truncated + std::vector b(r.bytes.begin(), r.bytes.begin() + cut); + b.insert(b.end(), r.bytes.begin() + 16 + payload, r.bytes.end()); + const uint64_t p = cut - 16; + memcpy(b.data() + 8, &p, 8); + attempt(b); + } + for (int k = 0; k < 300; k++) { // payload bytes changed + auto b = r.bytes; + for (int m = 0; m < 4; m++) b[16 + rng() % payload] ^= char(1 + rng() % 255); + attempt(b); + } + SUCCEED(); +} + +// Double values near the midpoint between two lattice points, where the double rounding of x / step and of q step +// back takes the most of the step's margin. +TEST(BioMD, DoubleNearHalfLatticePointsStaysWithinBound) { + const std::vector x = {510.39700369221237, 0, 0, 467.45074901980968, 0, 0}; + const double eb = 5.8345702187110699e-06; + const auto r = round_trip(x, 1, 2, eb); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD); + EXPECT_LE(r.max_err, eb); +} + +// A water whose O and H2 swap ends of the lattice between frames: the H2 symbol against the previous frame must +// still fit 32 bits. +template +void water_jumps_across_the_range() { + // the lattice spans |x| < 2^(LATTICE_SPAN_BITS + 1) eb when 2^LATTICE_SPAN_BITS eb is a power of two + const double eb = std::ldexp(1.0, -10), M = 0.99 * std::ldexp(eb, SZ3::biomd::LATTICE_SPAN_BITS + 1); + const size_t waters = 40, atoms = 3 * waters + 2, frames = 3; + std::vector x(frames * atoms * 3); + std::mt19937 rng(1); + std::uniform_real_distribution U(0, 3); + const double r = 0.09572, th = 104.52 * std::acos(-1.0) / 180; + std::vector O(3 * waters); + for (auto &v : O) v = U(rng); + for (size_t f = 0; f < frames; f++) { + T *X = &x[f * atoms * 3]; + for (size_t w = 0; w < waters; w++) { + double o[3] = {O[3 * w] + 0.001 * f, O[3 * w + 1], O[3 * w + 2]}; + double h1[3] = {o[0], o[1] + r, o[2]}, h2[3] = {o[0], o[1] + r * std::cos(th), o[2] + r * std::sin(th)}; + if (w == 7 && f > 0) o[0] = h1[0] = (f == 1 ? M : -M), h2[0] = (f == 1 ? -M : M); + for (int c = 0; c < 3; c++) + X[9 * w + c] = T(o[c]), X[9 * w + 3 + c] = T(h1[c]), X[9 * w + 6 + c] = T(h2[c]); + } + X[9 * waters] = T(M), X[9 * waters + 3] = T(-M); // two ions fix max|x| + } + const auto rt = round_trip(x, frames, atoms, eb); + EXPECT_EQ(rt.algo, SZ3::ALGO_BIOMD); + EXPECT_LE(rt.max_err, eb); +} +TEST(BioMD, WaterJumpsAcrossTheCoordinateRange) { + water_jumps_across_the_range(); + water_jumps_across_the_range(); +} + +// The O-H distance at 16384 - 1e-5 lattice steps: its square rounds to 2^28, which load() refuses, so the codec must +// not code the water on it. +TEST(BioMD, WaterGeometryAtTheLatticeLimitDecodes) { + SystemSpec s; + size_t n; + const auto x = make_system(s, &n); + const auto r = round_trip(x, s.frames, n, 3.1597940068971598e-06); + EXPECT_LE(r.max_err, 3.1597940068971598e-06); +} + +// An unwritten chunk is all fill, NaN included. +TEST(BioMD, ChunkOfFillOnly) { + const size_t frames = 3, atoms = 10; + const std::vector x(frames * atoms * 3, std::numeric_limits::quiet_NaN()); + const auto r = round_trip(x, frames, atoms, 5e-4); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD); + for (float v : r.out) EXPECT_TRUE(std::isnan(v)); +} + +// Few atoms and many frames: what each frame stores beyond its symbols must not outgrow the data. +TEST(BioMD, FewAtomsManyFrames) { + const size_t frames = 5000, atoms = 2; + std::vector x(frames * atoms * 3); + std::mt19937 rng(3); + std::normal_distribution step(0, 0.01f); + for (size_t i = 0; i < x.size(); i++) x[i] = (i < atoms * 3 ? 1.0f : x[i - atoms * 3]) + step(rng); + const auto r = round_trip(x, frames, atoms, 5e-4); + EXPECT_EQ(r.algo, SZ3::ALGO_BIOMD); + EXPECT_LE(r.max_err, 5e-4); +} + +} // namespace diff --git a/tools/test/modules/test_byte_util.cpp b/tools/test/modules/test_byte_util.cpp new file mode 100644 index 00000000..80852b9f --- /dev/null +++ b/tools/test/modules/test_byte_util.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +#include "SZ3/utils/ByteUtil.hpp" +#include "gtest/gtest.h" + +TEST(ByteUtilTest, Zigzag) { + const std::vector values = {0, -1, 1, -2, 2, INT64_MAX, INT64_MIN}; + const std::vector expected = {0, 1, 2, 3, 4, UINT64_MAX - 1, UINT64_MAX}; + for (size_t i = 0; i < values.size(); i++) { + EXPECT_EQ(SZ3::zigzag(values[i]), expected[i]); + EXPECT_EQ(SZ3::unzigzag(expected[i]), values[i]); + } +} + +TEST(ByteUtilTest, VarintRoundTrip) { + const std::vector values = {0, 1, 127, 128, 16383, 16384, uint64_t(1) << 35, UINT64_MAX}; + std::vector buf(10 * values.size()); + SZ3::uchar *w = buf.data(); + for (uint64_t v : values) SZ3::write_varint(v, w); + EXPECT_EQ(w - buf.data(), 1 + 1 + 1 + 2 + 2 + 3 + 6 + 10); + const SZ3::uchar *r = buf.data(); + size_t remaining = size_t(w - buf.data()); + for (uint64_t v : values) EXPECT_EQ(SZ3::read_varint(r, remaining), v); + EXPECT_EQ(remaining, 0u); +} + +TEST(ByteUtilTest, VarintRejectsTruncatedAndOverlong) { + const SZ3::uchar truncated[] = {0x80, 0x80}; + const SZ3::uchar *r = truncated; + size_t remaining = sizeof(truncated); + EXPECT_THROW(SZ3::read_varint(r, remaining), std::out_of_range); + + std::vector overlong(11, 0x80); + overlong.back() = 0; + r = overlong.data(); + remaining = overlong.size(); + EXPECT_THROW(SZ3::read_varint(r, remaining), std::out_of_range); + + std::vector past64(10, 0x80); // a 10th byte above 1 holds bits past 64 + past64.back() = 0x02; + r = past64.data(); + remaining = past64.size(); + EXPECT_THROW(SZ3::read_varint(r, remaining), std::out_of_range); +} + +TEST(ByteUtilTest, BitsRoundTrip) { + std::mt19937_64 gen(7); + std::vector> fields; + std::vector bytes; + SZ3::BitAppender appender(bytes); + size_t total_bits = 0; + for (int i = 0; i < 1000; i++) { + const int bits = int(gen() % 57); + const uint64_t v = bits == 0 ? 0 : gen() >> (64 - bits); + fields.emplace_back(v, bits); + appender.put(v, bits); + total_bits += bits; + } + appender.flush(); + EXPECT_EQ(bytes.size(), (total_bits + 7) / 8); + + SZ3::BitConsumer consumer(bytes.data(), bytes.data() + bytes.size()); + for (const auto &f : fields) EXPECT_EQ(consumer.get(f.second), f.first); + EXPECT_TRUE(consumer.at_end()); + EXPECT_THROW(consumer.get(8), std::out_of_range); +} diff --git a/tools/test/modules/test_config.cpp b/tools/test/modules/test_config.cpp new file mode 100644 index 00000000..2f2e9441 --- /dev/null +++ b/tools/test/modules/test_config.cpp @@ -0,0 +1,50 @@ +#include + +#include "SZ3/utils/Config.hpp" +#include "gtest/gtest.h" + +namespace { + +// A config as a later version would write it: three more bytes at the end, counted in its size byte. +std::vector with_appended_fields(const SZ3::Config &conf) { + std::vector buf(conf.size_est() + 16); + SZ3::uchar *c = buf.data(); + const size_t size = conf.save(c); + buf.resize(size); + buf.insert(buf.end(), {7, 7, 7}); + buf[0] = static_cast(size + 3); + return buf; +} + +} // namespace + +TEST(SZ3_Config, LoadSkipsFieldsAddedAtTheEnd) { + SZ3::Config conf(10, 20, 30); + conf.absErrorBound = 1e-3; + conf.openmp = true; + auto buf = with_appended_fields(conf); + buf.push_back(0xAB); // what follows the config, such as the next chunk's config in an OpenMP stream + + SZ3::Config loaded; + const SZ3::uchar *c = buf.data(); + size_t remaining = buf.size(); + loaded.load(c, remaining); + EXPECT_EQ(*c, 0xAB); + EXPECT_EQ(remaining, 1u); + EXPECT_EQ(loaded.num, conf.num); + EXPECT_EQ(loaded.absErrorBound, conf.absErrorBound); + EXPECT_TRUE(loaded.openmp); +} + +TEST(SZ3_Config, LoadRefusesASizeShorterThanItsFields) { + SZ3::Config conf(10, 20, 30); + std::vector buf(conf.size_est() + 16); + SZ3::uchar *c = buf.data(); + buf.resize(conf.save(c)); + buf[0] = 4; + + SZ3::Config loaded; + const SZ3::uchar *p = buf.data(); + size_t remaining = buf.size(); + EXPECT_THROW(loaded.load(p, remaining), std::out_of_range); +} diff --git a/tools/test/modules/test_decomposition_save_load.cpp b/tools/test/modules/test_decomposition_save_load.cpp new file mode 100644 index 00000000..fd11f032 --- /dev/null +++ b/tools/test/modules/test_decomposition_save_load.cpp @@ -0,0 +1,244 @@ +// The two halves of the save/load contract, for the decompositions the MD algorithms use. +// +// save() has to write the same bytes for the same input, or a compressed file cannot be +// checksummed and two runs of the same pipeline produce different output. +// +// A member that save() writes but some compress() path never assigns takes whatever the memory +// held before, so the check has to control what that was: the decomposition is constructed over +// a buffer filled with two different patterns, and the two headers compared. Compressing twice +// on the heap instead would only catch it when the allocator happens to hand back dirty memory. + +#include +#include +#include +#include +#include +#include + +#include "SZ3/decomposition/BlockwiseDecomposition.hpp" +#include "SZ3/decomposition/SZBioMDDecomposition.hpp" +#include "SZ3/predictor/ComposedPredictor.hpp" +#include "SZ3/predictor/LorenzoPredictor.hpp" +#include "SZ3/predictor/RegressionPredictor.hpp" +#include "SZ3/quantizer/LinearQuantizer.hpp" +#include "SZ3/utils/Config.hpp" +#include "gtest/gtest.h" + +namespace { + +std::vector noise(size_t count, uint32_t seed = 11) { + std::mt19937 rng(seed); + std::uniform_real_distribution spread(-5.0f, 5.0f); + std::vector data(count); + for (auto &value : data) { + value = spread(rng); + } + return data; +} + +/// A ramp is what a regression predictor fits best, so it is also what makes it store the most. +std::vector ramp(size_t count, uint32_t seed = 3) { + std::mt19937 rng(seed); + std::uniform_real_distribution jitter(-1.0f, 1.0f); + std::vector data(count); + for (size_t i = 0; i < count; i++) { + data[i] = static_cast(i) * 0.5f + jitter(rng); + } + return data; +} + +/// The compressor allocates from size_est() and save() writes without checking, so check the +/// bound directly: save with room to spare, so an overrun is this assertion, not a crash. +template +void expect_save_stays_within_size_est(Decomposition &decomposition, const SZ3::Config &conf, std::vector data, + const char *what) { + SZ3::Config work = conf; + decomposition.compress(work, data.data()); + + const size_t declared = decomposition.size_est(); + std::vector buffer(declared + (1u << 20)); + SZ3::uchar *cursor = buffer.data(); + decomposition.save(cursor); + + const size_t written = static_cast(cursor - buffer.data()); + EXPECT_LE(written, declared) << what << ": save() wrote " << written << " bytes where size_est() promised " + << declared; +} + +/// Compress over storage holding `poison`, and return what save() writes. +template +std::vector header_after_compress(const std::vector &dims, unsigned char poison, + MakeQuantizer make_quantizer) { + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 1e-3; + + std::vector storage(sizeof(Decomposition) + alignof(Decomposition)); + std::memset(storage.data(), poison, storage.size()); + void *place = storage.data(); + size_t room = storage.size(); + place = std::align(alignof(Decomposition), sizeof(Decomposition), place, room); + + auto data = noise(conf.num); + auto *decomposition = new (place) Decomposition(conf, make_quantizer(conf)); + decomposition->compress(conf, data.data()); + + std::vector header(decomposition->size_est() + 4096); + SZ3::uchar *cursor = header.data(); + decomposition->save(cursor); + header.resize(static_cast(cursor - header.data())); + decomposition->~Decomposition(); + return header; +} + +template +void expect_header_depends_only_on_input(const std::vector &dims, MakeQuantizer make_quantizer) { + const auto over_zeros = header_after_compress(dims, 0x00, make_quantizer); + const auto over_ones = header_after_compress(dims, 0xcd, make_quantizer); + ASSERT_EQ(over_zeros.size(), over_ones.size()); + EXPECT_EQ(over_zeros, over_ones) << "save() wrote bytes that came from the memory it was built over"; +} + +/// load() must charge remaining_length for exactly what it read, which is what keeps every +/// later parse inside the buffer. +template +void expect_load_charges_what_it_reads(const std::vector &dims, MakeQuantizer make_quantizer) { + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 1e-3; + + Decomposition writer(conf, make_quantizer(conf)); + auto data = noise(conf.num); + writer.compress(conf, data.data()); + + std::vector stream(writer.size_est() + 4096); + SZ3::uchar *write_cursor = stream.data(); + writer.save(write_cursor); + const size_t written = static_cast(write_cursor - stream.data()); + + Decomposition reader(conf, make_quantizer(conf)); + const SZ3::uchar *read_cursor = stream.data(); + size_t remaining = stream.size(); + reader.load(read_cursor, remaining); + + const size_t advanced = static_cast(read_cursor - stream.data()); + EXPECT_EQ(advanced, written) << "load() did not consume what save() wrote"; + EXPECT_EQ(stream.size() - remaining, advanced) + << "load() advanced " << advanced << " bytes but charged " << stream.size() - remaining; +} + +// SZBioMDDecomposition takes no quantizer: this one takes the helpers' second argument and drops it +template +struct BioMD : SZ3::SZBioMDDecomposition { + BioMD(const SZ3::Config &conf, int) : SZ3::SZBioMDDecomposition(conf) {} +}; +auto no_quantizer = [](const SZ3::Config &) { return 0; }; + +} // namespace + +TEST(SZ3_DecompositionSaveLoad, BioMDTwoDimensions) { + expect_header_depends_only_on_input>({777, 3}, no_quantizer); +} + +TEST(SZ3_DecompositionSaveLoad, BioMDThreeDimensions) { + expect_header_depends_only_on_input>({5, 777, 3}, no_quantizer); +} + +TEST(SZ3_DecompositionSaveLoad, BioMDChargesWhatItReads) { + expect_load_charges_what_it_reads>({777, 3}, no_quantizer); + expect_load_charges_what_it_reads>({5, 777, 3}, no_quantizer); +} + +// The mode bytes and the box flag hold 0 or 1; any other value is a corrupt stream, not a value to truncate. +TEST(SZ3_DecompositionSaveLoad, BioMDRefusesUnknownModeAndBoxFlag) { + const std::vector dims = {5, 777, 3}; + const auto header = header_after_compress>(dims, 0x00, no_quantizer); + // coded frames (8), fill value (4), step (8), water distances (8 + 8), water count (8), then 4 modes and the flag + const size_t modes = 44, box = modes + SZ3::biomd::NUM_GROUPS; + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + for (size_t at : {modes, modes + 3, box}) { + auto corrupt = header; + ASSERT_LE(corrupt[at], 1) << "byte " << at << " is not where the mode bytes and the box flag are"; + corrupt[at] = 2; + BioMD<3> reader(conf, 0); + const SZ3::uchar *cursor = corrupt.data(); + size_t remaining = corrupt.size(); + EXPECT_THROW(reader.load(cursor, remaining), std::runtime_error) << "byte " << at; + } +} + +// BlockwiseDecomposition declared nothing while save() serialised the regression coefficients, +// which grow as n/blockSize. Below the default block size that ran off the compressor's buffer. +TEST(SZ3_DecompositionSaveLoad, BlockwiseSizeEstBoundsSaveAtSmallBlockSizes) { + for (int blockSize : {2, 3, 4, 6, 8, 128}) { + for (double eb : {1e-1, 1e-2, 1e-3, 1e-4}) { + SZ3::Config conf(16384); + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + conf.blockSize = blockSize; + + std::vector>> predictors; + predictors.push_back(std::make_shared>(eb)); + predictors.push_back( + std::make_shared>(static_cast(blockSize), eb)); + + auto decomposition = + SZ3::make_decomposition_blockwise(conf, SZ3::ComposedPredictor(predictors), + SZ3::LinearQuantizer(eb, conf.quantbinCnt / 2)); + + SCOPED_TRACE("BlockSize=" + std::to_string(blockSize) + " eb=" + std::to_string(eb)); + expect_save_stays_within_size_est(decomposition, conf, ramp(conf.num), "BlockwiseDecomposition"); + } + } +} + +// A block RegressionPredictor declines falls back to Lorenzo, which reads a neighbour outside the block: with only +// the regression predictor's padding it read past the data in 3D and 4D. +template +void expect_regression_only_round_trip(const std::vector &dims) { + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + const double eb = 1e-3; + auto make = [&] { + return SZ3::make_decomposition_blockwise(conf, SZ3::RegressionPredictor(conf.blockSize, eb), + SZ3::LinearQuantizer(eb, conf.quantbinCnt / 2)); + }; + const auto input = noise(conf.num); + auto data = input; + auto writer = make(); + auto bins = writer.compress(conf, data.data()); + std::vector stream(writer.size_est()); + SZ3::uchar *write_cursor = stream.data(); + writer.save(write_cursor); + auto reader = make(); + const SZ3::uchar *read_cursor = stream.data(); + size_t remaining = write_cursor - stream.data(); + reader.load(read_cursor, remaining); + std::vector output(conf.num); + reader.decompress(conf, bins, output.data()); + for (size_t i = 0; i < conf.num; i++) { + ASSERT_LE(std::fabs(output[i] - input[i]), eb) << dims.size() << "D, value " << i; + } +} + +TEST(SZ3_DecompositionSaveLoad, BlockwiseRegressionOnlyFallback) { + expect_regression_only_round_trip<3>({40, 41, 43}); + expect_regression_only_round_trip<4>({12, 13, 14, 15}); +} + +TEST(SZ3_DecompositionSaveLoad, BioMDSizeEstBoundsSave) { + for (double eb : {1e-1, 1e-2, 1e-3, 1e-4}) { + SZ3::Config conf(5, 777, 3); + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + SCOPED_TRACE("eb=" + std::to_string(eb)); + + BioMD<3> biomd(conf, 0); + auto coordinates = ramp(conf.num); // up to 12 nm, within BIOMD's lattice (64 nm at 1e-4) + for (auto &v : coordinates) v *= 0.002f; + expect_save_stays_within_size_est(biomd, conf, coordinates, "SZBioMDDecomposition"); + } +} diff --git a/tools/test/modules/test_encoder.cpp b/tools/test/modules/test_encoder.cpp new file mode 100644 index 00000000..5597b24b --- /dev/null +++ b/tools/test/modules/test_encoder.cpp @@ -0,0 +1,48 @@ +#include +#include + +#include "SZ3/encoder/BypassEncoder.hpp" +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "gtest/gtest.h" + +template +void runFunctionalTest() { + int N = 1000; + std::vector buffer_data(N * sizeof(T) * 2); + std::vector buffer_conf(N * sizeof(T) * 2); + std::vector data(N); + for (int i = 0; i < N; i++) { + data[i] = i % 100; + } + + size_t data_len = 0, conf_len = 0; + { + SZ3::uchar *buffer_data_pos = buffer_data.data(); + SZ3::uchar *buffer_conf_pos = buffer_conf.data(); + Encoder coder; + coder.preprocess_encode(data, 100); + coder.encode(data, buffer_data_pos); + coder.save(buffer_conf_pos); + data_len = buffer_data_pos - buffer_data.data(); + conf_len = buffer_conf_pos - buffer_conf.data(); + } + { + const SZ3::uchar *buffer_data_pos = buffer_data.data(); + const SZ3::uchar *buffer_conf_pos = buffer_conf.data(); + Encoder coder; + coder.load(buffer_conf_pos, conf_len); + auto dataDecoded = coder.decode(buffer_data_pos, N, data_len); + for (int i = 0; i < N; i++) { + EXPECT_EQ(data[i], dataDecoded[i]); + } + } +} + +template +void runAllTest() { + runFunctionalTest(); +} + +TEST(EncoderTest, HuffmanEncoder) { runAllTest, int>(); } + +TEST(EncoderTest, BypassEncoder) { runAllTest, int>(); } diff --git a/tools/test/modules/test_huffman_encoder.cpp b/tools/test/modules/test_huffman_encoder.cpp new file mode 100644 index 00000000..f5fcd1ab --- /dev/null +++ b/tools/test/modules/test_huffman_encoder.cpp @@ -0,0 +1,435 @@ +// HuffmanEncoder: exact round trips, save() == size_est(), size_bound() covers save() + encode(), +// optimal payloads, stable bytes, and decoders that throw rather than read out of bounds on corrupted streams. +// +// Every buffer handed to save(), encode(), load() and decode() is an exactly sized heap allocation, so a sanitizer +// build reports an access one byte past what the stream holds. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/utils/MemoryUtil.hpp" +#include "gtest/gtest.h" + +namespace { + +using SZ3::uchar; + +template +struct Encoded { + std::vector tree, data; + size_t est = 0; +}; + +template +Encoded encode(const std::vector &bins) { + Encoded e; + SZ3::HuffmanEncoder enc; + enc.preprocess_encode(bins, 0); + e.est = enc.size_est(); + { + std::unique_ptr t(new uchar[e.est]); + uchar *p = t.get(); + enc.save(p); + e.tree.assign(t.get(), p); + } + // First into a roomy buffer to learn the size, then again into an exact one. + size_t size; + { + std::vector big(32 + bins.size() * 5); + uchar *p = big.data(); + size = enc.encode(bins, p); + EXPECT_EQ(size, static_cast(p - big.data())); + } + std::unique_ptr d(new uchar[size]); + uchar *p = d.get(); + EXPECT_EQ(enc.encode(bins, p), size); + e.data.assign(d.get(), p); + enc.postprocess_encode(); + return e; +} + +template +std::vector decode(const Encoded &e, size_t n) { + std::unique_ptr t(new uchar[e.tree.size()]); + std::unique_ptr d(new uchar[e.data.size()]); + if (!e.tree.empty()) memcpy(t.get(), e.tree.data(), e.tree.size()); + if (!e.data.empty()) memcpy(d.get(), e.data.data(), e.data.size()); + SZ3::HuffmanEncoder dec; + const uchar *tp = t.get(); + size_t trem = e.tree.size(); + dec.load(tp, trem); + EXPECT_EQ(trem, 0u); + dec.preprocess_decode(); + const uchar *dp = d.get(); + size_t drem = e.data.size(); + auto out = dec.decode(dp, n, drem); + dec.postprocess_decode(); + EXPECT_EQ(drem, 0u); + return out; +} + +// The bit count encode() writes in front of the codes. +uint64_t payload_bits(const std::vector &v) { + uint64_t bits = 0; + const uchar *p = v.data(); + SZ3::read(bits, p); + return bits; +} + +// Cost in bits of an optimal prefix code for these bins, and the depth of one optimal tree. +template +uint64_t optimal_bits(const std::vector &bins, int *depth = nullptr) { + std::map freq; + for (T b : bins) freq[b]++; + if (depth) *depth = 0; + if (freq.size() < 2) return 0; + std::priority_queue, std::vector>, std::greater<>> q; + for (auto &kv : freq) q.push({kv.second, 0}); + uint64_t cost = 0; + int d = 0; + while (q.size() > 1) { + auto a = q.top(); + q.pop(); + auto b = q.top(); + q.pop(); + cost += a.first + b.first; + d = std::max(a.second, b.second) + 1; + q.push({a.first + b.first, d}); + } + if (depth) *depth = d; + return cost; +} + +template +size_t distinct(std::vector v) { + std::sort(v.begin(), v.end()); + return std::unique(v.begin(), v.end()) - v.begin(); +} + +// Everything a valid stream must satisfy. +template +Encoded check(const std::vector &bins, bool expect_optimal = true) { + Encoded e = encode(bins); + EXPECT_EQ(e.tree.size(), e.est) << "save() must write exactly size_est()"; + EXPECT_LE(e.tree.size() + e.data.size(), SZ3::HuffmanEncoder::size_bound(bins.size(), distinct(bins))); + const uint64_t bits = payload_bits(e.data); + EXPECT_EQ(e.data.size(), sizeof(uint64_t) + (bits + 7) / 8); + if (expect_optimal) EXPECT_EQ(bits, optimal_bits(bins)); + auto out = decode(e, bins.size()); + EXPECT_TRUE(out == bins) << "round trip differs, n = " << bins.size(); + return e; +} + +std::vector laplace(size_t n, double scale, int center, uint64_t seed) { + std::mt19937_64 g(seed); + std::exponential_distribution ex(1.0 / scale); + std::vector v(n); + for (size_t i = 0; i < n; i++) { + double x = ex(g); + v[i] = center + static_cast(std::lround((g() & 1) ? x : -x)); + } + return v; +} + +TEST(SZ3_HuffmanEncoder, Empty) { + std::vector bins; + auto e = check(bins); + EXPECT_EQ(e.tree.size(), sizeof(uint32_t)); + EXPECT_EQ(e.data.size(), sizeof(uint64_t)); + // An empty table cannot produce values. + SZ3::HuffmanEncoder dec; + const uchar *tp = e.tree.data(); + size_t trem = e.tree.size(); + dec.load(tp, trem); + const uchar *dp = e.data.data(); + size_t drem = e.data.size(); + EXPECT_THROW(dec.decode(dp, 1, drem), std::out_of_range); +} + +TEST(SZ3_HuffmanEncoder, SingleSymbol) { + for (int v : {0, 1, -1, 32768, INT_MAX, INT_MIN}) { + for (size_t n : {1, 2, 1000}) { + auto e = check(std::vector(n, v)); + EXPECT_EQ(e.data.size(), sizeof(uint64_t)); + } + } +} + +TEST(SZ3_HuffmanEncoder, SmallAlphabetsEveryBitBoundary) { + std::mt19937_64 g(1); + for (int k = 2; k <= 9; k++) { + for (size_t n = 1; n <= 130; n++) { + std::vector bins(n); + for (auto &b : bins) b = 100 + static_cast(g() % k); + check(bins); + } + } +} + +TEST(SZ3_HuffmanEncoder, QuantizationLike) { + for (double scale : {0.3, 1.0, 5.0, 60.0, 3000.0}) { + for (size_t n : {10, 1000, 7956, 100000}) { + auto bins = laplace(n, scale, 32768, static_cast(scale * 1000) + n); + for (size_t i = 0; i < n; i += 97) bins[i] = 0; // unpredictable + check(bins); + } + } +} + +// A dominant symbol gets the one-bit all-zero code, and the decoder takes its runs in one step. +TEST(SZ3_HuffmanEncoder, LongRunsOfTheCommonestSymbol) { + std::mt19937_64 g(16); + for (int rarity : {3, 20, 200, 5000}) { + for (size_t n : {2, 9, 63, 64, 65, 200, 4099, 60000}) { + std::vector bins(n, 32768); + for (auto &b : bins) + if (g() % rarity == 0) b = 32768 + static_cast(g() % 9) - 4; + check(bins); + bins.assign(n, 7); + bins.back() = 8; // a run up to the last code + check(bins); + bins.assign(n, 7); + bins.front() = -8; + check(bins); + } + } +} + +// Table-length codes and longer ones, around every length the decoder switches on. +TEST(SZ3_HuffmanEncoder, CodeLengthsAroundTheTable) { + for (int top = 8; top <= 22; top++) { + std::vector bins; + // Geometric weights 2^-k give code lengths 1..top. + for (int k = 0; k <= top; k++) + for (int j = 0; j < (1 << (top - k)); j++) bins.push_back(k * 3 - 20); + std::shuffle(bins.begin(), bins.end(), std::mt19937_64(top)); + check(bins); + } +} + +TEST(SZ3_HuffmanEncoder, NegativeAndExtremeBins) { + check(std::vector{INT_MIN, INT_MAX, INT_MIN, 0, -1, 1, INT_MAX}); + check(laplace(5000, 20, -100000, 3)); + std::vector w{INT64_MIN, INT64_MAX, 0, INT64_MIN, -5, 5}; + check(w); + std::vector u{0, UINT64_MAX, UINT64_MAX, 12345}; + check(u); +} + +TEST(SZ3_HuffmanEncoder, OtherTypes) { + std::mt19937_64 g(4); + std::vector a(3000); + for (auto &x : a) x = static_cast(g() % 7 + 250 * (g() % 50 == 0)); + check(a); + std::vector b(3000); + for (auto &x : b) x = static_cast(static_cast(g() % 11) - 5 + (g() % 100 == 0 ? -32768 : 0)); + check(b); + std::vector c(3000); + for (auto &x : c) x = static_cast(g() % 3 ? 65535 : g() % 40); + check(c); +} + +// A range far wider than the input takes the hash path. +TEST(SZ3_HuffmanEncoder, SparseWideRange) { + std::mt19937_64 g(5); + std::vector vals(3000); + for (auto &x : vals) x = static_cast(g()); + std::geometric_distribution geo(0.003); + for (size_t n : {2, 50, 4000, 200000}) { + std::vector bins(n); + for (auto &b : bins) b = vals[std::min(geo(g), 2999)]; + check(bins); + } + // All distinct. + std::vector all(20000); + for (size_t i = 0; i < all.size(); i++) all[i] = static_cast(i * 104729u) - 1000000000; + check(all); +} + +// Fibonacci weights: an optimal code is 34 bits deep, so the lengths are limited to kMaxLen. +TEST(SZ3_HuffmanEncoder, LengthLimited) { + std::vector bins; + uint64_t a = 1, b = 1; + for (int k = 0; k < 35; k++) { + bins.insert(bins.end(), a, k); + uint64_t c = a + b; + a = b; + b = c; + } + std::shuffle(bins.begin(), bins.end(), std::mt19937_64(6)); + int depth; + const uint64_t opt = optimal_bits(bins, &depth); + ASSERT_GT(depth, 32); + auto e = check(bins, false); + const uint64_t bits = payload_bits(e.data); + EXPECT_GE(bits, opt); + EXPECT_LE(bits, opt + opt / 1000000); +} + +// The same bins give the same bytes on every host; these were produced by this implementation. +TEST(SZ3_HuffmanEncoder, StableBytes) { + std::vector bins{5, 5, 5, 7, 7, 9, 5, 12, 5, 7}; + auto e = check(bins); + // Lengths 5:1 7:2 9:3 12:3, so codes 0, 10, 110, 111 and 17 payload bits. After the count 4 and the offset 5 + // (little-endian), the table holds gaps 2 2 3 and length changes +1 +1 +1 0. + const std::vector tree{4, 0, 0, 0, 5, 0, 0, 0, 0x69, 0xa6, 0xe0}; + const std::vector data{17, 0, 0, 0, 0, 0, 0, 0, 0x15, 0x9d, 0x00}; + EXPECT_EQ(e.tree, tree); + EXPECT_EQ(e.data, data); + // Independent of what the encoder was used for before. + SZ3::HuffmanEncoder enc; + enc.preprocess_encode(laplace(1000, 3, 0, 7), 0); + enc.preprocess_encode(bins, 0); + std::vector t(enc.size_est()); + uchar *p = t.data(); + enc.save(p); + EXPECT_EQ(t, tree); +} + +TEST(SZ3_HuffmanEncoder, CopiesAreIndependent) { + auto bins = laplace(3000, 4, 10, 8); + SZ3::HuffmanEncoder enc; + enc.preprocess_encode(bins, 0); + auto copy = enc; + enc.preprocess_encode(std::vector{1, 2, 3}, 0); + std::vector buf(copy.size_est() + 16 + bins.size() * 4); + uchar *p = buf.data(); + copy.save(p); + copy.encode(bins, p); + SZ3::HuffmanEncoder dec; + const uchar *c = buf.data(); + size_t rem = p - buf.data(); + dec.load(c, rem); + EXPECT_EQ(dec.decode(c, bins.size(), rem), bins); +} + +TEST(SZ3_HuffmanEncoder, RefusesBinsItWasNotBuiltFor) { + SZ3::HuffmanEncoder enc; + enc.preprocess_encode(std::vector{1, 3, 3}, 0); + std::vector buf(64); + uchar *p = buf.data(); + EXPECT_THROW(enc.encode(std::vector{1, 3, 9}, p), std::invalid_argument); // outside the range + p = buf.data(); + EXPECT_THROW(enc.encode(std::vector{1, 2, 3}, p), std::invalid_argument); // inside, never counted + enc.preprocess_encode(std::vector{4, 4}, 0); + p = buf.data(); + EXPECT_THROW(enc.encode(std::vector{4, 5}, p), std::invalid_argument); // one symbol + enc.preprocess_encode(std::vector{}, 0); + p = buf.data(); + EXPECT_THROW(enc.encode(std::vector{0}, p), std::invalid_argument); // no symbol + enc.preprocess_encode(std::vector{0, 1000000, 1000000}, 0); + p = buf.data(); + EXPECT_THROW(enc.encode(std::vector{0, 1000000, 7}, p), std::invalid_argument); // hashed +} + +TEST(SZ3_HuffmanEncoder, WrongValueCountThrows) { + auto bins = laplace(5000, 6, 0, 9); + auto e = encode(bins); + for (size_t n : {size_t(0), size_t(1), bins.size() - 1, bins.size() + 1, bins.size() * 40}) { + SZ3::HuffmanEncoder dec; + const uchar *tp = e.tree.data(); + size_t trem = e.tree.size(); + dec.load(tp, trem); + const uchar *dp = e.data.data(); + size_t drem = e.data.size(); + EXPECT_THROW(dec.decode(dp, n, drem), std::out_of_range) << n; + } +} + +// Decode a possibly corrupted stream from exactly sized buffers; true if it decoded. +bool try_decode(const std::vector &tree, const std::vector &data, size_t n) { + std::unique_ptr te(new uchar[tree.size()]); + std::unique_ptr de(new uchar[data.size()]); + if (!tree.empty()) memcpy(te.get(), tree.data(), tree.size()); + if (!data.empty()) memcpy(de.get(), data.data(), data.size()); + try { + SZ3::HuffmanEncoder dec; + const uchar *tp = te.get(); + size_t trem = tree.size(); + dec.load(tp, trem); + if (tp > te.get() + tree.size() || trem > tree.size()) ADD_FAILURE() << "load overran"; + const uchar *dp = de.get(); + size_t drem = data.size(); + auto out = dec.decode(dp, n, drem); + if (out.size() != n) ADD_FAILURE() << "decode returned " << out.size() << " values"; + if (dp > de.get() + data.size() || drem > data.size()) ADD_FAILURE() << "decode overran"; + return true; + } catch (const std::out_of_range &) { + return false; + } catch (const std::length_error &) { + return false; + } catch (const std::bad_alloc &) { + ADD_FAILURE() << "allocation driven by the stream"; + return false; + } +} + +TEST(SZ3_HuffmanEncoder, TruncationsThrow) { + for (auto bins : {laplace(3000, 2, 32768, 10), laplace(3000, 300, 32768, 11)}) { + auto e = encode(bins); + for (size_t k = 0; k < e.tree.size(); k++) + EXPECT_FALSE(try_decode(std::vector(e.tree.begin(), e.tree.begin() + k), e.data, bins.size())); + for (size_t k = 0; k < e.data.size(); k++) + EXPECT_FALSE(try_decode(e.tree, std::vector(e.data.begin(), e.data.begin() + k), bins.size())); + } +} + +TEST(SZ3_HuffmanEncoder, MutationsNeverOverrun) { + std::mt19937_64 g(12); + std::vector> inputs{laplace(2000, 1, 32768, 13), laplace(2000, 50, 32768, 14), + laplace(300, 3000, 0, 15), std::vector(100, 7)}; + { + std::vector fib; // long codes + uint64_t a = 1, b = 1; + for (int k = 0; k < 20; k++) { + fib.insert(fib.end(), a, k * 1000); + uint64_t c = a + b; + a = b; + b = c; + } + inputs.push_back(fib); + } + size_t decoded = 0, total = 0; + for (auto &bins : inputs) { + auto e = encode(bins); + for (int it = 0; it < 3000; it++) { + auto t = e.tree; + auto d = e.data; + int flips = 1 + static_cast(g() % 4); + for (int f = 0; f < flips; f++) { + bool in_tree = (g() & 1) && !t.empty(); + auto &v = in_tree ? t : d; + if (v.empty()) continue; + switch (g() % 3) { + case 0: + v[g() % v.size()] ^= static_cast(1u << (g() % 8)); + break; + case 1: + v[g() % v.size()] = static_cast(g()); + break; + default: + if (v.size() > 1) v.resize(g() % v.size()); + break; + } + } + total++; + decoded += try_decode(t, d, bins.size()); + } + } + // Some mutations still decode (a flipped payload bit can map to another valid code); none may overrun. + RecordProperty("decoded", static_cast(decoded)); + RecordProperty("total", static_cast(total)); +} + +} // namespace diff --git a/tools/test/modules/test_integer_data.cpp b/tools/test/modules/test_integer_data.cpp new file mode 100644 index 00000000..401c45af --- /dev/null +++ b/tools/test/modules/test_integer_data.cpp @@ -0,0 +1,70 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" +#include "gtest/gtest.h" + +// Every decompressed integer lies within floor(bound) of the input, including values at the type's limits and +// data whose range the type cannot hold. +template +void runIntegers() { + const T lo = std::numeric_limits::lowest(), hi = std::numeric_limits::max(); + std::mt19937_64 g(7); + std::vector> inputs(3, std::vector(20000)); + for (size_t i = 0; i < 20000; i++) { + inputs[0][i] = static_cast(g()); // full range + inputs[1][i] = (g() & 1) ? static_cast(hi - T(g() % 4)) : static_cast(lo + T(g() % 4)); // both ends + inputs[2][i] = static_cast(hi - T((i * 7) % 50)); // smooth, at the top + } + if (sizeof(T) == 8) { // within +-2^53 + for (auto &in : inputs) + for (auto &v : in) v = static_cast(v >> 12); + } + for (const auto &in : inputs) { + for (int algo : {SZ3::ALGO_LORENZO_REG, SZ3::ALGO_INTERP_LORENZO, SZ3::ALGO_INTERP, SZ3::ALGO_NOPRED}) { + for (auto e : {std::make_pair(SZ3::EB_ABS, 0.0), std::make_pair(SZ3::EB_ABS, 1.0), + std::make_pair(SZ3::EB_ABS, 3.7), std::make_pair(SZ3::EB_REL, 1e-2)}) { + SZ3::Config conf(in.size()); + conf.cmprAlgo = algo; + conf.errorBoundMode = e.first; + conf.absErrorBound = e.second; + conf.relErrorBound = e.second; + size_t cmpSize = 0; + std::unique_ptr cmp(SZ_compress(conf, in.data(), cmpSize)); + SZ3::Config dconf; + std::unique_ptr out(SZ_decompress(dconf, cmp.get(), cmpSize)); + const double range = + double(*std::max_element(in.begin(), in.end())) - double(*std::min_element(in.begin(), in.end())); + const double bound = std::floor(e.first == SZ3::EB_ABS ? e.second : e.second * range); + for (size_t i = 0; i < in.size(); i++) { + ASSERT_LE(std::fabs(double(out[i]) - double(in[i])), bound) + << "algo " << algo << " mode " << e.first << " bound " << e.second << " at " << i; + } + } + } + } +} + +TEST(SZ3_IntegerData, WithinTheBound) { + runIntegers(); + runIntegers(); + runIntegers(); + runIntegers(); + runIntegers(); + runIntegers(); + runIntegers(); + runIntegers(); +} + +TEST(SZ3_IntegerData, EightByteValuesPast2To53AreRefused) { + std::vector in(100, int64_t(1) << 53); + SZ3::Config conf(in.size()); + size_t cmpSize = 0; + EXPECT_THROW(SZ_compress(conf, in.data(), cmpSize), std::invalid_argument); +} diff --git a/tools/test/modules/test_lossless.cpp b/tools/test/modules/test_lossless.cpp new file mode 100644 index 00000000..50042029 --- /dev/null +++ b/tools/test/modules/test_lossless.cpp @@ -0,0 +1,46 @@ +#include +#include +#include + +#include "SZ3/lossless/Lossless_bypass.hpp" +#include "SZ3/lossless/Lossless_zstd.hpp" +#include "gtest/gtest.h" + +template +void runFunctionalTest() { + Lossless lossless; + size_t N = 1000; + std::vector src(N); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis(0, 255); + for (size_t i = 0; i < N; i++) { + src[i] = static_cast(dis(gen)); + } + std::vector dst(SZ3::Lossless_zstd::compress_bound(src.size()) + sizeof(size_t)); + size_t compressedSize = lossless.compress(src.data(), src.size(), dst.data(), dst.size()); + + std::vector decompressed(N); + SZ3::uchar* decompressed_pos = decompressed.data(); + size_t decompressedSize = lossless.decompress(dst.data(), compressedSize, decompressed_pos, decompressed.size()); + + EXPECT_EQ(decompressedSize, src.size()); + EXPECT_EQ(std::vector(decompressed.data(), decompressed.data() + decompressedSize), src); +} + +template +void runAllTest() { + runFunctionalTest(); +} + +TEST(LosslessTest, LosslessZstd) { runAllTest(); } +TEST(LosslessTest, LosslessBypass) { runAllTest(); } + +// A buffer too small for even the size header is refused, not written past. +TEST(LosslessTest, LosslessZstdRefusesBufferSmallerThanHeader) { + std::vector src(100, 7); + for (size_t cap = 0; cap < sizeof(size_t); cap++) { + std::vector dst(cap); + EXPECT_THROW(SZ3::Lossless_zstd().compress(src.data(), src.size(), dst.data(), cap), std::length_error); + } +} diff --git a/tools/test/modules/test_previous_data_version.cpp b/tools/test/modules/test_previous_data_version.cpp new file mode 100644 index 00000000..67e534c0 --- /dev/null +++ b/tools/test/modules/test_previous_data_version.cpp @@ -0,0 +1,101 @@ +// Streams SZ3 wrote in data version 3.3.2, which this build still reads. +// +// tools/test/streams// holds tools/sz3/testfloat_8_8_128.dat compressed by that release's sz3 CLI +// (-3 8 8 128, ABS 1e-3 unless the name says otherwise; the _omp ones with OpenMP on and 4 threads), and each +// decode must match that release's decode bit for bit. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" +#include "gtest/gtest.h" + +namespace { + +std::vector slurp(const std::string &name) { + std::ifstream in(std::string(SZ3_TEST_STREAMS) + "/3.3.2/" + name + ".sz", std::ios::binary); + return std::vector(std::istreambuf_iterator(in), std::istreambuf_iterator()); +} + +// FNV-1a over the decoded bytes. +uint64_t fnv1a(const float *data, size_t n) { + uint64_t h = 0xcbf29ce484222325ull; + const auto *b = reinterpret_cast(data); + for (size_t i = 0; i < n * sizeof(float); i++) h = (h ^ b[i]) * 0x100000001b3ull; + return h; +} + +TEST(SZ3_PreviousDataVersion, DecodesAsItsReleaseDid) { + const std::pair streams[] = { + {"interp", 0xe49ee9e883e32415ull}, + {"interp_omp", 0x1172c536c4c77855ull}, + {"lorenzo_reg", 0x3f7750a23094223dull}, + {"lorenzo_reg_omp", 0xe32514629ac5fd3dull}, + {"lorenzo_reg_rel1e-5", 0xe0885e9322ea80ccull}, + {"lossless", 0x51c93bc7f98c12a5ull}, + }; + for (const auto &s : streams) { + const auto cmp = slurp(s.first); + ASSERT_FALSE(cmp.empty()) << s.first; + SZ3::Config conf; + std::vector dec(8 * 8 * 128); + float *p = dec.data(); + SZ_decompress(conf, cmp.data(), cmp.size(), p); + EXPECT_EQ(conf.sz3DataVer, versionInt("3.3.2")) << s.first; + EXPECT_EQ(fnv1a(dec.data(), dec.size()), s.second) << s.first; + } +} + +// That version's ALGO_NOPRED and ALGO_BIOMD streams are not supported, nor the removed ALGO_BIOMDXTC. +TEST(SZ3_PreviousDataVersion, NopredAndBioMDAreRefused) { + for (const char *name : {"nopred", "biomd", "biomdxtc"}) { + const auto cmp = slurp(name); + ASSERT_FALSE(cmp.empty()) << name; + SZ3::Config conf; + std::vector dec(8 * 8 * 128); + float *p = dec.data(); + EXPECT_THROW(SZ_decompress(conf, cmp.data(), cmp.size(), p), std::invalid_argument) << name; + } +} + +// Outside SZ3_DATA_VER_OLDEST to SZ3_DATA_VER, a stream is refused with the way to read it. +TEST(SZ3_PreviousDataVersion, VersionsOutsideTheRangeAreRefused) { + auto cmp = slurp("lorenzo_reg"); + ASSERT_FALSE(cmp.empty()); + for (const auto &v : {std::make_pair(versionInt("3.3.0"), "Use SZ3 v3.3.0"), + std::make_pair(versionInt(SZ3_DATA_VER) + (1u << 8), "Upgrade SZ3")}) { + auto pos = reinterpret_cast(cmp.data() + 4); // after the magic number + SZ3::write(v.first, pos); + SZ3::Config conf; + std::vector dec(8 * 8 * 128); + float *p = dec.data(); + try { + SZ_decompress(conf, cmp.data(), cmp.size(), p); + ADD_FAILURE() << versionStr(v.first) << " was read"; + } catch (const std::invalid_argument &e) { + EXPECT_NE(std::string(e.what()).find(v.second), std::string::npos) << e.what(); + } + } +} + +// Compressing with a Config loaded from 3.3.2 data writes this build's data version, which is what the payload is. +TEST(SZ3_PreviousDataVersion, RecompressingWritesThisVersion) { + const auto cmp = slurp("lorenzo_reg"); + ASSERT_FALSE(cmp.empty()); + SZ3::Config conf; + std::unique_ptr dec(SZ_decompress(conf, cmp.data(), cmp.size())); + size_t n = 0; + std::unique_ptr again(SZ_compress(conf, dec.get(), n)); + SZ3::Config back; + std::unique_ptr out(SZ_decompress(back, again.get(), n)); + EXPECT_EQ(back.sz3DataVer, versionInt(SZ3_DATA_VER)); +} + +} // namespace diff --git a/tools/test/modules/test_quantizer.cpp b/tools/test/modules/test_quantizer.cpp new file mode 100644 index 00000000..e75d1e30 --- /dev/null +++ b/tools/test/modules/test_quantizer.cpp @@ -0,0 +1,67 @@ +#include +#include + +#include "gtest/gtest.h" +#include "SZ3/quantizer/LinearQuantizer.hpp" + +template +void runQuantizeRecoverTest() { + T eb = 12.1973; + Quantizer quantizer(eb); + T data = static_cast(100.11212); + T data_ori = data; + + int quant_index = quantizer.quantize_and_overwrite(data, static_cast(0)); + + T recovered = quantizer.recover(static_cast(0), quant_index); + + EXPECT_NEAR(recovered, data_ori, eb); +} + +template +void runFunctionalTest() { + T eb = 12.1973; + Quantizer quantizer(eb); + + const int N = 100; + std::vector originals(N); + std::vector quant_indices(N); + + // Generate 100 different input values. + // We choose values that vary slightly so that quantization succeeds. + for (int i = 0; i < N; i++) { + T data = static_cast(10.723 + (i * 0.0005)); + originals[i] = data; + quant_indices[i] = quantizer.quantize_and_overwrite(data, static_cast(0)); + } + + // Save the quantizer's state to a temporary buffer. + std::vector buffer(N * sizeof(T) * 4); + unsigned char* save_ptr = buffer.data(); + quantizer.save(save_ptr); + size_t saved_size = save_ptr - buffer.data(); + EXPECT_GT(saved_size, 0u) << "Saved state must be non-empty."; + + // Create a new quantizer instance and load the saved state. + Quantizer loadedQuantizer(eb); // The error bound will be overwritten by load(). + const unsigned char* load_ptr = buffer.data(); + size_t remaining_size = saved_size; + loadedQuantizer.load(load_ptr, remaining_size); + + // Now, recover each value using the stored quantization indices. + // Verify that the recovered value is within the error bound of the original. + for (int i = 0; i < N; i++) { + T recovered = loadedQuantizer.recover(static_cast(0), quant_indices[i]); + EXPECT_NEAR(recovered, originals[i], eb) << "Mismatch at index " << i; + } +} + +template +void runAllTest() { + runQuantizeRecoverTest(); + runFunctionalTest(); +} + +TEST(QuantizerTest, LinearQuantizer) { + runAllTest, float>(); +} diff --git a/tools/test/modules/test_reconstruction.cpp b/tools/test/modules/test_reconstruction.cpp new file mode 100644 index 00000000..45ec79fd --- /dev/null +++ b/tools/test/modules/test_reconstruction.cpp @@ -0,0 +1,123 @@ +// The compressor predicts from its own reconstructed values, so the decompressor must reconstruct exactly the +// same values. Built with FMA (-march=x86-64-v3), this fails if only one of the two is protected by nofma(). +// ALGO_BIOMD leaves the input as it was and predicts on its integer lattice: the values it decodes, compressed again, +// must decode to themselves. + +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" +#include "gtest/gtest.h" + +namespace { + +template +size_t differing_values(SZ3::ALGO algo, SZ3::INTERP_ALGO interp, double eb) { + SZ3::Config conf = algo == SZ3::ALGO_BIOMD ? SZ3::Config(32, 341, 3) : SZ3::Config(32, 32, 32); + conf.cmprAlgo = algo; + conf.interpAlgo = interp; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + std::vector data(conf.num), out(conf.num); + for (size_t i = 0; i < conf.num; i++) { + double x = i / 1024, y = i / 32 % 32, z = i % 32; + data[i] = static_cast(std::sin(0.3 * x) * std::cos(0.2 * y) + 0.5 * std::sin(0.1 * z + 0.05 * x)); + } + + std::vector cmp(SZ3::SZ_compress_size_bound(conf)); + if (algo == SZ3::ALGO_INTERP) { + size_t size = SZ3::SZ_compress_Interp(conf, data.data(), cmp.data(), cmp.size()); + SZ3::SZ_decompress_Interp(conf, cmp.data(), size, out.data()); + } else { + size_t size = SZ3::SZ_compress_bioMD(conf, data.data(), cmp.data(), cmp.size()); + if (conf.cmprAlgo != SZ3::ALGO_BIOMD) return 0; // beyond its lattice: stored losslessly + SZ3::SZ_decompress_bioMD(conf, cmp.data(), size, data.data()); + size = SZ3::SZ_compress_bioMD(conf, data.data(), cmp.data(), cmp.size()); + SZ3::SZ_decompress_bioMD(conf, cmp.data(), size, out.data()); + } + size_t differ = 0; + for (size_t i = 0; i < conf.num; i++) { + differ += std::memcmp(&data[i], &out[i], sizeof(T)) != 0; + } + return differ; +} + +template +void expect_identical(SZ3::ALGO algo, SZ3::INTERP_ALGO interp) { + for (double eb : {1e-2, 1e-3, 1e-4, 1e-5, 1e-6}) { + EXPECT_EQ(differing_values(algo, interp, eb), 0u) << "eb = " << eb; + } +} + +} // namespace + +TEST(SZ3_Reconstruction, InterpLinear) { + expect_identical(SZ3::ALGO_INTERP, SZ3::INTERP_ALGO_LINEAR); + expect_identical(SZ3::ALGO_INTERP, SZ3::INTERP_ALGO_LINEAR); +} + +TEST(SZ3_Reconstruction, InterpCubic) { + expect_identical(SZ3::ALGO_INTERP, SZ3::INTERP_ALGO_CUBIC); + expect_identical(SZ3::ALGO_INTERP, SZ3::INTERP_ALGO_CUBIC); +} + +TEST(SZ3_Reconstruction, BioMD) { + expect_identical(SZ3::ALGO_BIOMD, SZ3::INTERP_ALGO_CUBIC); + expect_identical(SZ3::ALGO_BIOMD, SZ3::INTERP_ALGO_CUBIC); +} + +// SZ_compress leaves the input as it was: BIOMD and LORENZO_REG only read it (LORENZO_REG, with any of its +// predictors, works on a padded copy), the other algorithms get a copy. +TEST(SZ3_Reconstruction, SZCompressLeavesTheInputAsItWas) { + struct Case { + SZ3::ALGO algo; + bool lorenzo = true, lorenzo2 = false, regression = true; + }; + for (const Case &c : + {Case{SZ3::ALGO_LORENZO_REG}, Case{SZ3::ALGO_LORENZO_REG, true, false, false}, + Case{SZ3::ALGO_LORENZO_REG, false, true, false}, Case{SZ3::ALGO_LORENZO_REG, false, false, true}, + Case{SZ3::ALGO_LORENZO_REG, true, true, true}, Case{SZ3::ALGO_INTERP_LORENZO}, Case{SZ3::ALGO_INTERP}, + Case{SZ3::ALGO_NOPRED}, Case{SZ3::ALGO_BIOMD}}) { + for (const std::vector &dims : {std::vector{32, 341, 3}, std::vector{5000}, + std::vector{70, 73}, std::vector{5, 6, 7, 9}}) { + if (c.algo == SZ3::ALGO_BIOMD && dims.back() != 3) continue; // coordinates only + SZ3::Config conf; + conf.setDims(dims.begin(), dims.end()); + conf.cmprAlgo = c.algo; + conf.lorenzo = c.lorenzo, conf.lorenzo2 = c.lorenzo2, conf.regression = c.regression; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = 1e-3; + std::vector data(conf.num); + for (size_t i = 0; i < conf.num; i++) data[i] = static_cast(std::sin(0.01 * i) + 0.1 * (i % 3)); + const std::vector before = data; + size_t size = 0; + delete[] SZ_compress(conf, data.data(), size); + EXPECT_EQ(std::memcmp(before.data(), data.data(), data.size() * sizeof(float)), 0) + << c.algo << " " << c.lorenzo << c.lorenzo2 << c.regression << " " << dims.size() << "D"; + } + } +} + +// An error bound that is negative or not finite leaves nothing to bound: every algorithm compresses losslessly, as with +// a bound of 0. +TEST(SZCompress, BoundsThatAreNotPositiveAndFiniteGoLossless) { + for (SZ3::ALGO algo : + {SZ3::ALGO_LORENZO_REG, SZ3::ALGO_INTERP_LORENZO, SZ3::ALGO_INTERP, SZ3::ALGO_NOPRED, SZ3::ALGO_BIOMD}) { + for (double eb : {0.0, -1e-3, double(INFINITY), double(NAN)}) { + SZ3::Config conf(40, 30, 3); + conf.cmprAlgo = algo; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = eb; + std::vector data(conf.num); + for (size_t i = 0; i < conf.num; i++) data[i] = static_cast(std::sin(0.01 * i)); + size_t size = 0; + std::unique_ptr cmp(SZ_compress(conf, data.data(), size)); + SZ3::Config dec; + std::unique_ptr out(SZ_decompress(dec, cmp.get(), size)); + EXPECT_EQ(dec.cmprAlgo, SZ3::ALGO_LOSSLESS) << algo << " " << eb; + EXPECT_EQ(std::memcmp(out.get(), data.data(), data.size() * sizeof(float)), 0) << algo << " " << eb; + } + } +} diff --git a/tools/test/modules/test_segmented_encoder.cpp b/tools/test/modules/test_segmented_encoder.cpp new file mode 100644 index 00000000..75b07eed --- /dev/null +++ b/tools/test/modules/test_segmented_encoder.cpp @@ -0,0 +1,89 @@ +// SegmentedEncoder: bins that are a sequence of segments [n, n bins], each with a Huffman code of its own. + +#include +#include +#include +#include +#include +#include + +#include "SZ3/encoder/HuffmanEncoder.hpp" +#include "SZ3/encoder/SegmentedEncoder.hpp" +#include "gtest/gtest.h" + +namespace { + +std::vector segmented_round_trip(const std::vector &bins) { + SZ3::SegmentedEncoder> enc(64); + enc.preprocess_encode(bins, 0); + std::vector buf(enc.size_est() + 8 * bins.size() + 1024); + SZ3::uchar *p = buf.data(); + enc.save(p); + enc.encode(bins, p); + const size_t n = p - buf.data(); + SZ3::SegmentedEncoder> dec(std::numeric_limits::max()); + const SZ3::uchar *q = buf.data(); + size_t rem = n; + dec.load(q, rem); + return dec.decode(q, bins.size(), rem); +} + +TEST(SZ3_SegmentedEncoder, RoundTrip) { + std::mt19937 g(5); + std::vector bins; + for (int s = 0; s < 30; s++) { + const int n = s % 3 == 0 ? 0 : int(g() % 6000); // empty segments too + bins.push_back(n); + for (int i = 0; i < n; i++) bins.push_back(int(g() % (s + 2)) - s / 2); + } + EXPECT_EQ(segmented_round_trip(bins), bins); + EXPECT_EQ(segmented_round_trip({}), std::vector()); + EXPECT_EQ(segmented_round_trip({0, 0, 0}), std::vector({0, 0, 0})); +} + +// Corrupt segment tables: sizes beyond an int, sizes that do not add up (even with wrap-around), and many empty +// segments, which must cost their bytes rather than an encoder each. +TEST(SZ3_SegmentedEncoder, CorruptSegmentTables) { + auto stream = [](uint32_t count, std::vector rest) { + std::vector b(4); + memcpy(b.data(), &count, 4); + b.insert(b.end(), rest.begin(), rest.end()); + return b; + }; + auto load = [](const std::vector &b, SZ3::SegmentedEncoder> &dec) { + const SZ3::uchar *p = b.data(); + size_t rem = b.size(); + dec.load(p, rem); + return rem; + }; + { + SZ3::SegmentedEncoder> dec(std::numeric_limits::max()); + EXPECT_THROW(load(stream(1, {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}), dec), + std::out_of_range); // 2^64 - 1 + EXPECT_THROW(load(stream(1, {0x80, 0x80, 0x80, 0x80, 0x08}), dec), std::out_of_range); // 2^31 + } + { + SZ3::SegmentedEncoder> dec(std::numeric_limits::max()); + load(stream(3, {0, 0, 0}), dec); + const SZ3::uchar *p = nullptr; + size_t rem = 0; + EXPECT_THROW(dec.decode(p, 2, rem), std::out_of_range); // three counts are three bins + EXPECT_EQ(dec.decode(p, 3, rem), std::vector({0, 0, 0})); + } + { + const uint32_t count = 1000000; + SZ3::SegmentedEncoder> dec(std::numeric_limits::max()); + load(stream(count, std::vector(count, 0)), dec); + const SZ3::uchar *p = nullptr; + size_t rem = 0; + EXPECT_EQ(dec.decode(p, count, rem).size(), count); + } + { + // more segments than the caller allows: refused before an encoder is built for each + SZ3::SegmentedEncoder> dec(12); + EXPECT_THROW(load(stream(13, std::vector(13 * 5, 0)), dec), std::out_of_range); + EXPECT_NO_THROW(load(stream(12, std::vector(12, 0)), dec)); + } +} + +} // namespace diff --git a/tools/test/openmpThreadCounts.sh b/tools/test/openmpThreadCounts.sh new file mode 100755 index 00000000..cac497c5 --- /dev/null +++ b/tools/test/openmpThreadCounts.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# An OpenMP stream carries the thread count it was written with, and the reader asks the runtime +# for that many. It may get fewer, and then it must still decode every chunk to the same bytes. +# +# tools/test/openmpThreadCounts.sh +# tools/test/openmpThreadCounts.sh build/tools/sz3/sz3 input.dat 1 -3 128 8 8 +# +# Only OMP_THREAD_LIMIT can force the reader to get fewer: it caps the whole program, so a +# num_threads request cannot exceed it. OMP_NUM_THREADS cannot -- num_threads overrides it -- so +# setting that on the reader would leave the count unchanged and test nothing. +# +# Assert the bytes, never just that the exit was 0. No set -e: every check below is explicit, and +# the total at the bottom catches a section that stopped early. +set -u +SZ3=$(cd "$(dirname "$1")" && pwd)/$(basename "$1") +INPUT=$(cd "$(dirname "$2")" && pwd)/$(basename "$2") +BOUND=$3 +shift 3 +DIMS=("$@") +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# Required, not skipped: a missing binary must never look like a passing check. +if [ ! -x "$SZ3" ]; then echo "no sz3 binary at $SZ3"; exit 1; fi +if [ ! -f "$INPUT" ]; then echo "no input file at $INPUT"; exit 1; fi +if [ ${#DIMS[@]} -eq 0 ]; then echo "no dimension arguments given"; exit 1; fi + +cd "$WORK" || exit 1 +printf '[GlobalSettings]\nOpenMP = YES\n' > omp.config + +pass=0; fail=0 +ok() { echo "PASS $1"; pass=$((pass+1)); } +bad() { echo "FAIL $1"; shift; for l in "$@"; do echo " $l"; done; fail=$((fail+1)); } + +echo "=== OpenMP thread counts: $SZ3 ===" +echo " input $INPUT" +echo " dims ${DIMS[*]} bound $BOUND" + +WRITERS="1 4 8" +LIMITS="1 2 3" + +for w in $WRITERS; do + if OMP_NUM_THREADS=$w "$SZ3" -f -i "$INPUT" "${DIMS[@]}" -M ABS "$BOUND" \ + -c omp.config -z "omp$w.sz3" > "w$w.log" 2>&1 && [ -s "omp$w.sz3" ]; then + ok "omp-$w-thread-stream-written" + else + bad "omp-$w-thread-stream-written" "$(tail -5 "w$w.log")" + fi + # The reference this stream's own reader produces, before any cap is put on it. + if "$SZ3" -f -s "omp$w.sz3" -o "omp$w.ref.dat" "${DIMS[@]}" > "r$w.log" 2>&1; then + ok "omp-$w-thread-stream-reads-back" + else + bad "omp-$w-thread-stream-reads-back" "$(tail -5 "r$w.log")" + fi + for l in $LIMITS; do + if OMP_THREAD_LIMIT=$l "$SZ3" -f -s "omp$w.sz3" -o "omp$w.$l.dat" "${DIMS[@]}" \ + > "r$w.$l.log" 2>&1; then + ok "omp-$w-thread-stream-reads-under-limit-$l" + else + bad "omp-$w-thread-stream-reads-under-limit-$l" "$(tail -5 "r$w.$l.log")" + fi + if [ -s "omp$w.ref.dat" ] && cmp -s "omp$w.ref.dat" "omp$w.$l.dat"; then + ok "omp-$w-thread-stream-identical-under-limit-$l" + else + bad "omp-$w-thread-stream-identical-under-limit-$l" \ + "OMP_THREAD_LIMIT=$l changed the data a $w-thread stream decodes to" + fi + done +done + +# Without this the suite is 24 checks on a library that was built with no OpenMP in it: the config +# key is ignored, every stream is the plain serial one, and all of them round-trip perfectly. +differ=0 +for w in 4 8; do + cmp -s omp1.sz3 "omp$w.sz3" || differ=1 +done +if [ "$differ" = 1 ]; then + ok "thread-count-changes-the-stream" +else + bad "thread-count-changes-the-stream" \ + "1, 4 and 8 writer threads produced the same bytes, so this binary has no OpenMP path" \ + "$(ls -l omp1.sz3 omp4.sz3 omp8.sz3 2>&1)" +fi + +echo +echo " $pass passed, $fail failed" + +# Raise this with the check it comes with. A loop that stopped early otherwise shows only as a +# smaller number at the bottom that nobody compares. +EXPECTED=25 +ran=$((pass + fail)) +if [ "$ran" -ne "$EXPECTED" ]; then + echo " the suite accounted for $ran checks, not $EXPECTED" + exit 1 +fi +exit $((fail > 0)) diff --git a/tools/test/openmp_regions.cpp b/tools/test/openmp_regions.cpp new file mode 100644 index 00000000..2c562563 --- /dev/null +++ b/tools/test/openmp_regions.cpp @@ -0,0 +1,72 @@ +// conf.openmp = true inside a program that runs its own OpenMP. +// +// openmp_regions throw one chunk throws: the call must throw, not hang or terminate +// openmp_regions nested compress from each thread of the program's own region; with nested +// parallelism on and OMP_THREAD_LIMIT set, regions get fewer threads +// than SZ3 asked for, and every stream must still decode inside the bound + +#include + +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" + +int main(int argc, char** argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + if (mode == "throw") { + // ALGO_BIOMD refuses every chunk of this 4D data, whose last dimension is not 3. + SZ3::Config conf(5, 8, 8, 8); + conf.cmprAlgo = SZ3::ALGO_BIOMD; + conf.absErrorBound = 1e-3; + conf.openmp = true; + std::vector data(conf.num, 1.0f); + omp_set_num_threads(4); + try { + size_t cmpSize = 0; + delete[] SZ_compress(conf, data.data(), cmpSize); + } catch (const std::invalid_argument& e) { + printf("PASS threw: %s\n", e.what()); + return 0; + } + fprintf(stderr, "FAIL a chunk that cannot be compressed did not throw\n"); + return 1; + } + if (mode == "nested") { + SZ3::Config base(32, 64, 64); + base.absErrorBound = 1e-3; + base.openmp = true; + std::vector data(base.num); + for (size_t i = 0; i < base.num; i++) data[i] = static_cast(10 * std::sin(0.001 * i)); + int failures = 0, rounds = 0; +#pragma omp parallel num_threads(4) reduction(+ : failures, rounds) + for (int round = 0; round < 250; round++, rounds++) { + try { + SZ3::Config conf = base; + size_t cmpSize = 0; + std::unique_ptr cmpData(SZ_compress(conf, data.data(), cmpSize)); + SZ3::Config readConf; + std::unique_ptr dec(SZ_decompress(readConf, cmpData.get(), cmpSize)); + for (size_t i = 0; i < conf.num; i++) { + if (std::fabs(static_cast(dec[i]) - data[i]) > conf.absErrorBound * (1 + 1e-6)) { + failures++; + break; + } + } + } catch (const std::exception&) { + failures++; + } + } + if (failures) { + fprintf(stderr, "FAIL %d round trips threw or left the bound\n", failures); + return 1; + } + printf("PASS %d round trips inside the bound\n", rounds); + return 0; + } + fprintf(stderr, "usage: %s \n", argv[0]); + return 2; +} diff --git a/tools/test/openmp_stream.cpp b/tools/test/openmp_stream.cpp new file mode 100644 index 00000000..7524c198 --- /dev/null +++ b/tools/test/openmp_stream.cpp @@ -0,0 +1,152 @@ +// One stream compressed with conf.openmp = true, decoded by a build of the other kind. +// +// openmp_stream write [biomd] compress, write .sz3 and its decode .dec +// openmp_stream read decode .sz3, compare with .dec bit for bit +// +// biomd writes ALGO_BIOMD water {frames, atoms, 3}, which the chunks split along the atoms, inside a molecule. +// +// tools/test/CMakeLists.txt builds this twice, with OpenMP and without, and has the build without it +// read what the other wrote. The first argument says which build this is meant to be, so a build that got OpenMP +// when it should not have (or the reverse) fails instead of testing nothing. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SZ3/api/sz.hpp" + +static std::vector slurp(const std::string& path) { + std::ifstream in(path, std::ios::binary); + return std::vector(std::istreambuf_iterator(in), std::istreambuf_iterator()); +} + +static void spill(const std::string& path, const char* data, size_t size) { + std::ofstream(path, std::ios::binary).write(data, static_cast(size)); +} + +int main(int argc, char** argv) { + if (argc != 4 && argc != 5) { + fprintf(stderr, "usage: %s [biomd]\n", argv[0]); + return 2; + } + const std::string kind = argv[1], mode = argv[2], prefix = argv[3]; + const bool biomd = argc == 5 && std::string(argv[4]) == "biomd"; +#ifdef _OPENMP + const std::string built = "with"; +#else + const std::string built = "without"; +#endif + if (kind != built) { + fprintf(stderr, "FAIL this binary was built %s OpenMP, not %s it\n", built.c_str(), kind.c_str()); + return 1; + } + + try { + if (mode == "write") { + // 1003 waters and one ion, 3010 atoms: 3 of the 4 chunks (at atoms 752, 1505, 2257) start inside a water + SZ3::Config conf = biomd ? SZ3::Config(20, 3010, 3) : SZ3::Config(64, 32, 32); + conf.cmprAlgo = biomd ? SZ3::ALGO_BIOMD : SZ3::ALGO_INTERP_LORENZO; + conf.errorBoundMode = SZ3::EB_ABS; + conf.absErrorBound = biomd ? 5e-4 : 1e-3; + conf.openmp = true; + std::vector data(conf.num); + for (size_t i = 0; i < conf.num; i++) { + const double x = static_cast(i); + data[i] = static_cast(std::sin(0.01 * x) + 0.001 * static_cast(i % 97)); + } + if (biomd) { + // O, then H1 and H2 0.09572 nm from it at 104.52 degrees; each frame turns and moves every water + const size_t atoms = conf.dims[1]; + for (size_t f = 0; f < conf.dims[0]; f++) { + for (size_t w = 0; w * 3 + 2 < atoms; w++) { + float* o = &data[(f * atoms + w * 3) * 3]; + const double a = 0.1 * static_cast(f) + 0.7 * static_cast(w); + const double half = 0.5 * 104.52 * 3.14159265358979 / 180; + for (int c = 0; c < 3; c++) + o[c] = static_cast(0.31 * static_cast((w * (7 + c)) % 31) + + 0.002 * static_cast(f) * (c + 1)); + for (int h = 0; h < 2; h++) { + const double t = a + (h ? half : -half); + o[3 + 3 * h] = static_cast(o[0] + 0.09572 * std::cos(t)); + o[4 + 3 * h] = static_cast(o[1] + 0.09572 * std::sin(t)); + o[5 + 3 * h] = o[2]; + } + } + } + } + size_t cmpSize = 0; + char* cmpData = SZ_compress(conf, data.data(), cmpSize); + std::vector dec(conf.num); + float* decPtr = dec.data(); + SZ3::Config readConf; + SZ_decompress(readConf, cmpData, cmpSize, decPtr); + for (size_t i = 0; i < conf.num; i++) { + if (std::fabs(dec[i] - data[i]) > conf.absErrorBound) { + fprintf(stderr, "FAIL value %zu comes back past the bound\n", i); + delete[] cmpData; + return 1; + } + } + // The payload of a chunked stream starts, after the 16-byte header, with its chunk count. + int32_t chunks = 0; + if (readConf.openmp) { + const SZ3::uchar* pos = reinterpret_cast(cmpData) + 16; + SZ3::read(chunks, pos); + // biomd: the first chunk keeps all frames, so it was split along the atoms + SZ3::Config first; + size_t remaining = cmpSize - 20; + first.load(pos, remaining); + if (biomd && first.dims[0] != conf.dims[0]) { + fprintf(stderr, "FAIL the first chunk holds %zu of %zu frames\n", first.dims[0], conf.dims[0]); + delete[] cmpData; + return 1; + } + } + printf("wrote %s.sz3: %zu bytes, openmp flag %d, %d chunks\n", prefix.c_str(), cmpSize, + static_cast(readConf.openmp), static_cast(chunks)); +#ifdef _OPENMP + // One chunk would decode the same through either path and prove nothing. + if (!readConf.openmp || chunks < 2) { + fprintf(stderr, "FAIL an OpenMP build wrote no multi-chunk stream; set OMP_NUM_THREADS above 1\n"); + delete[] cmpData; + return 1; + } +#endif + spill(prefix + ".sz3", cmpData, cmpSize); + spill(prefix + ".dec", reinterpret_cast(dec.data()), dec.size() * sizeof(float)); + delete[] cmpData; + } else if (mode == "read") { + std::vector cmp = slurp(prefix + ".sz3"); + std::vector want = slurp(prefix + ".dec"); + if (cmp.empty() || want.empty()) { + fprintf(stderr, "FAIL nothing to read at %s.sz3 and %s.dec\n", prefix.c_str(), prefix.c_str()); + return 1; + } + SZ3::Config conf; + float* dec = nullptr; + SZ_decompress(conf, cmp.data(), cmp.size(), dec); + const size_t got = conf.num * sizeof(float); + const bool same = got == want.size() && memcmp(dec, want.data(), got) == 0; + delete[] dec; + if (!same) { + fprintf(stderr, "FAIL %s.sz3 decodes to other bits here than where it was written\n", + prefix.c_str()); + return 1; + } + printf("read %s.sz3: openmp flag %d, %zu bytes identical\n", prefix.c_str(), + static_cast(conf.openmp), got); + } else { + fprintf(stderr, "unknown mode %s\n", mode.c_str()); + return 2; + } + } catch (const std::exception& e) { + fprintf(stderr, "FAIL %s\n", e.what()); + return 1; + } + return 0; +} diff --git a/tools/test/streams/3.3.2/biomd.sz b/tools/test/streams/3.3.2/biomd.sz new file mode 100644 index 0000000000000000000000000000000000000000..d72c638d42ead49b369b352cf7cb66a2875d0cbe GIT binary patch literal 870 zcmV-s1DX5~^Fs3g0s{jx0{{R30001J2LJ#70000ewJ-f(Xa)@#02C?!0001h0Ab>) zJ_7uy@zPCVP(J_w01Y8`P7uq0;p~WW{B?o5(QQO-L}eHpu@&V-X_4NDx2P`i7wL-v z02qrzEpZWAg6zPz@Go#Z+!S_;eFGAY1H^)o^|1P+Kz5>{2w_oKx+jQ{{3000s+pP6R=pE36T&lrOcLiqnr2ofZS|A>f)g#7KcOZFKBAfwK-z(KUYVWr>AdV`2SBM&^cUMKHg`H{r@w@AcPS9{}X})3F7}h zA|fIo|IEnP)ZEg_`v2d_&C}b-pFFC7{{PbmbPku6kM|j4j4=oy zg#Z8lgdjnJ`2YWph=@qY|NlQTGB!20w6b?{^Yr%eaCQE7w70W$aB=td2l!ct6y@b* zq@)#;@&Er*O?*IBo|DD`Cy}dkKo&WzG z?d@zGT-<&C|NQ}e79vG?x&MC|DQN{|{Qp!F9}wmL9}KinQ&E!S|Nobe6{)Ig2*Urr zps7XI=7`1r|6Cnio}Rve;r~C4K<992`FN?H{i&b*sh|D-|Ns9FPN-27L=171dR_KA zK!X`d(g6}c!OPy{M9csG(*ae64uq#pqKH(w<6)4f+mR0A8I{(m-uML@k;7r^3U6E$ zv)cOZ5FoX7f~`gp=uodH5J{9O-mTUB8;i;=9P)wW#-vAwx8fxhywH2;Z6t@F!p`}U wqH4=S7@Vr4Py{0b2!IF(03ZMW000001pxf1@zPCVP(Ppm000311^@s60~r{vyZ`_I literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/biomdxtc.sz b/tools/test/streams/3.3.2/biomdxtc.sz new file mode 100644 index 0000000000000000000000000000000000000000..885d90e1e7e26dcc3c596f95da6386c71fad32c2 GIT binary patch literal 8949 zcmchd2{=^y|Hsc5W8ac(Y}v9$mSic}85JgklwBmdY>`HU?E9q#lclmtWH0+3DwL}v z*(zB^qWyQyh;r|Jp8IdE=Re<>4wf_2 zi2`MkFZoN4j@lR;`u_bp`Hwn~0`852bJA$ zMf$KEv-|Ij(p}D?%#6@gauRj9BKKPG<=biU61LWftPJV&IgPymIeQ2~# zN}-_~wym_nZ!z+0HnC&-=`;n7&H2+-_#GoMW#mJ_u%P(z%Dx!;-{*5}y?5Q9)~ z%f6KYN@LmTmK9l3j<%2Cb~$v37;!Skb|on9Y^83L&c3lfo^6U4!~$y^+v#ZVX36@s z5S7-xxkwSFr(FfPjXC=E7tUykMx3ZvLM>($8xV_}Htm_I|H?^n?3#BWnjiI9HtOub zqn~K+e(X%hW>ZrtAKA+nMl3>i-u_ln@b~!dofjt$;dgzc{}TR9{ksHI&C!8x84#M- zw2rFcxspCzI$6ZAAuUBpLP%Q6hS$uBtNRrHK#BqYCck3j0OxO9*Z@9W2 zIxs&`Jh$vvGjy@|K+ZHTLiNa$xKedtz=1y{ye;2)jiL(FdvTKIAL%EQKkPN@ zP6+iyuyG!c*>#>)Hfhqi!i&W-*P5$1#iOu`GKsu?4~4`Kb8xHZ6UjygviPlo3M#5G zgz{4S+~`6zt$zaH-+8w#^WRZ-Wa$Csg+>P|%iZ@@*_d+b#mTupGW%Xu`J69+b3nQG zlp)<@%GyCpCcep?Td)-eqWs(jj?P*2CSz2zpMCcRJQV3u(Kf#wRfg18=;!gMa1-uF zdqmHY;vC2hTsKhA4j8TO=x$z62)j@hR^U4>WLoy3IKtCI&v5~bPIlYaEGQqh^R7+^-h4lwSU*-3k>Res!p3m{`z@kNujpxe@v`B)gJ zeiMNodoMrUnMFB)7)PQFIXY=PF(w@}r#@y-ci(vBW61;Q2%nE(8oLEHgjF3IkLmFPL5$CiW>f-jxV?nVn zk)yRkw-R!{Ue&D3yz!N6G;mUf7(|aaAvuJPq`P~CS2~6DYX(yfn#V`~jq?S8M#0w{ zh+7G}hRk_tUZ-Yj7##i?!8Ju48nEsrKG^|caU!ILaKoX({G(RnrVluRZaP`sx?=1*P}Lp#Cx80DgvRq z|7C76CeufH2#)E&e9S#(6-JUr;rvI3LslNv?@&yf@W%)}-An1+roJA>KloRid~<(z z=5CRF0XZeowp_S%5->n_iq?yBT<2?BRh^MVu|Iz2Txip4v67p`Z=aKvCum86QAh|h zetL(s;fzq{h~u>Q6iSqGQX^K=i#wW*Tdmmh3z_>FVC#U2=YCZcP+thOt=JiW@a~_N zjUNK@cNCV2FN8iw6)&K#SvG{cHynWp7PUdH0d+21rzlDdCFu)xbbuY4YR z-X3(^y)rJvs3wTSyMAVw>J*38GUs%-5Yu$*`9my4Xct}co<xJ{@zs8aR1dSP35 zE>?u?Nx5(?F$i^QP~}mfxkr17R`$=zjn43CMOahJ_t-T?B5frzm*2Zn>beXUYs3(P z%-lF}MP8C8{53gfOg1lJP*qVDvvoj4@eL0n0xC!%+qZ%dME{XWM!t~ZY}Mpj_vdCo z23~&A(Dt<;Y`@M^(6am{qCqo$e@@cO!YaM%6nRUxNa%pNdclo+@4h=8WJY$9b=(do zFF$KfxJ3+7=&hf_s5U>Ap@=VMe`h{naQ&L=1aLWR4+2dmRadcj{Vr7JHhjxjNOTguIQwh& zs(BhQ^&Pvdbnp4#hABxQ#q$Z6hI*PT}9#LPAEhv z9D}+L`HeN$TrNUyvURa!G5-yn8d)Tq^}RneCL>G+^S~b)MvdV*4|O>5EbzE5;rBF8 zy)-!F(5%$Zjv(|svY2lFKL4Lx4%GO_v%=HymqLP`_i``e?30q&IiSu*@oHE`SdM>> zf27xXmjg9E^6sOQu*JGQ>nzuoYi^^}{oPRKqc|3w;j+;TK{(X-$k(h<1$PHD1>bj_ zSO~C^?&^j*A4Tr-6BmCrLtz+DQ2>5sICe%H!!WdM7G;akA*E;cPTOXSvSD;e^+Y@D zw%4L;F*nJ#_0)Skx(u@)ieC>?8>^WdOS+}RWmosVWe+a6TWNVPlKc+b9Kk%CS!Sj$7nNc1>2bHR4FjYSEp?GqcOGQV7=NoeSZTY* zKwhVZD?v+$hvZTm>%Jx11oAP@KMq!k&ljgSn?;RPB)xQfisrTErrw#Reb(Nb#fG3K z;^LO)xa~C(ij{}E?~1oAoIovCYFc?vRrV|*|GG*fQ1LIT#0b?MZdpuNE&*)Z+S>Ad zPy8E-B|F=X2v|fm|Iy_JYK8CuHLQam9?EY}!pKJ_rRT`Zd*`G;taoznN%AYVg+a8l z1Jnwk1?JUt@p!N66~AM6&Ss9xu+OWL(}km3X7Hq~B1{iSAsGafd0Ha(eCmc&TTwo}}N zqyJ7Ps2Z^L0SB=Z3xvo58QIsmsD^i#o;4i+Cd>o_9Z4`%%^bf96sItI)wH%4-KLXg z4Of9(Za6ilx;W@@nm*!kD9LXkje`wz^3p=he!+fZMn>&;I46#Isf9JjK@Fc@Jvs=V z`?y;)K;XE|;@k|lroht>jR9qI~2L)d0v^OMZ@?AsjGfHDRGe<0fx{4an zlvStD_AwVEHRwSGu4d(6bPZiAodnJtI-TI+kbhtG?h6G*q7dl+fBe_*1iI+;K-Jkt zz?K1rie(#UHt0xnj^t^6chHm`)PdrZU8ud3wOUP10@^FbJD~_Bfr-n7k>)oM>OIx@ zwd`ER`wo|QUyG2=;l1#5S#eTgS~f=YeD^YGpYE(OH~NU+C1MbA%eBK@%ymYG@*;KY zIx#4ib<&c8p+h>EYrMTr@sbEt`1{)^fl_Qb10^rzzs>PGWJFi0j7yIzpW$NT#pJ7T z>|Pp~dF{YW|H>BBmYs>X8=k3L&x{)~U04-MS8CoBBAtSpwM^$Cr4#@rCQqtTYO-0K z+ZP7>A24cFd^Q*T+2n|z{%FL}*;z9C5TxXXHPOA~zk@&FWJeA>JihSceHfNqW$O?F ziy6O!DNw9JsBOjARiGkvJu2I@IJ-Xn18QGi@C-y50wV*OaY6l|z5&rx=rI;}1rK9z z#?cn`MnofBgQB{9eP0W*Q3fo6XhT5S!5NhvAz@iL{V}lxn5BI%mrcuwnznY$nh>q2 z12YGL+iS~eV4TXqnwU9kcF<~-M^*}UkL{GPge4}!P2o%PA6gIVI~1;WwJGE@@ViK3 zumZ3)%(sp9=-o5hDc7|TfJ>^TRz2XrJMg?MD!bKBAE>V?!f*<}9w?Z@QuT}^yX&b8 z$7pCVpiN<{X@L|ZzIfXL*)6dyVtWJGEwL`H8s;~(y@BjztgE)PtJrXR1KG`37YaV~ zs5^N-ss0+<5vnmfv|EM(h#uZ(kGdfR3ERFvb|cnJfXhpM-sV7dGuC}>INKUuR|UC~ zu)5=G$-);8GbQSN>`DL<8B&plcBaVr$De*pX5#dMG5*%S&n>^d6GvHMm**&z$N`Cl zp#&w9jVB;13RiZyhhbBxKd@;!&yjl1TJ!Lqn8=5DF~V5BE5ww?~NvBx9Wk2 zE6hF8n1;6~Wvw1Oea+Tx?Ifv z4fP2moS=bQZCuDYdR)fBf%nolxkdT_rAi+H^XfbALV}5(5&g(w?1D$I8B~qok#rN#_6237O(KsE5!@a2TBCnrC+D(y#O+3b8VsBi(X?<(VbIb-iJ9aq7 zaSSCx2E=m8HLCzmZA3+{Do4oNCP$cnL3jWu2B4D&BEflPE?R-@ zQQg>6nsU#&OcoWo@XYG0CR@j`3zK~p9e8`jX`b!1 zC>usc>~ZAnk?pl8TaC^(Ta+zE2TDW!n)tz>;$ITK%TVp)dg6B$P!d{^Q-7ZXT+4N4 z$=0}N?0g9dTXv`pKji5)vixw)aF0gt$f5k(zl(&3O0zzXlPOV~x78VBEh^o6WU?gW zMu&I-kvc-tM_j^kh>&l#fFOhT?61;QzWs|1&%_0^234Z$=gppe7t|H)+? zU{{x$EWlhg4{r`?g|1R_qsO{<)F$eZo%@5P4n|01K}>;$6*l1v=CXNsDNrlAKDS*M zCx}M_BX*M1yN<06Mo45iwD8d@tZaQyglpT)3{UoLsEZCPS{Dzen_w;*Mf>R~A(35S zMrz?tao`5oNIXiAEb!Yu+M`#y#73(hKC}b8*{AJc>xnEi805|o?h|`dAiJkbf^~mp zE8oDpMHc};w}{nB0w@OPyi!#VC+9u(4vDMBB3=#v)@~-=41xPN%|a%4t%2V~_DQP8 z(p*{0fj#-g zh}U!-qJM3Zhy)`M6(X|O%?wv(WClt}XsYS2YHLwgi5i4;1^W1ZGV2IgdkSgGT1F@@@%l9{ftsAiWA=-}@O0L; pQ|nT)ZcP$#nHoBiCOrIb3Je9s#?Mp8fQ=uqK$l|}Kn8B%{{y&B=rRBR literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/interp.sz b/tools/test/streams/3.3.2/interp.sz new file mode 100644 index 0000000000000000000000000000000000000000..fabe46dc29254815192298bb5386c5def589bdbc GIT binary patch literal 1740 zcmV;-1~d5(^Fs3g0s{k?1^@s60001e5dZ)H0000ewJ-f(eGok~018#46FMN;t2NWZ z%KhgA!D84bh4Bpr-1d}T0k_ZnGB+-76oeU!WbU1M!%j5IR>A(52H8@HJ62SQH_!li$tVsNCnBpk@!4f}-bRXXy+WzzYyVsrn5dY^sc=$WO zad`XPezspN4NZKqF|l6&qcGu@@Mnep2>&76!PL(n3hD{GK*9IF_h;%Z{V(v}=tcj~ zg%0%d^IW^5lk?mB0cZeVXRg-oWc@LIsyQ$E*x!ED7n|1-)EQ&0R#jbo>woUEw*yz$ z#;HCmUqI$iF9kHUMQV|<+deE>8d#{8m^b7|!p5sVZdgkLD^dsZCMW^Xf(Lb)g-l3J zNtGr*c8l;KdRI8&?4(h)0jA#fJ>-hErzC+;Y=#T!Q!LDD)j;CfUd@k#_up!@-fFd4 zvHW(Ije+K`xBPfbeh^!9zdiWd4sz=Hw*jR8{yeqcM*VFeJio22%fHR&JHA5mB5_S=5iA8lik z|GO1a85*WEaI~*g8Z}c%?fAe zFvAB5#qvdE)c5U14o-a?J*w<7Q6o$5mYp>q21`oZlGKDK193$IRkT7J%dC-lpmYq( zPb!TuHg;ora8pf9LWo6PiTq*$hFTkKCo3x}f3|ez^Vu(*POskEt(lpb*>-zApU+~k zSPTZEt+=OdZf})q2HdwOlLv^fqc>YjKa}E@*W{ z`XO|mrS<7rV?Tzo**U|k)jG&?#I*!N!ak{Tf;(+6CM_5e)Vli##AKTkdm+0#TvWl} zQ89Lsb%e%6wMgP+h_~fN!9fnM6!7R!%8pzJU%Z4Wwzv zRobE2kt7mHLJU)6Kt5i1ZKYWZ+CUZka1|zT6(&7;z6D8}6nlXJG1&keyMT_}Efr&D z6;&{JQ7w{e&=DGUZ&G?UDZM^fU_M!3PeX!QkDhNq(j=W^S-rb?L@_o081=vu!|+1a zeFuQIsDYUxL@6ROA|fJ5(iQ`70zz}N?GixLM2{iG2oWekL_#ExK;kBlu!u^b;c%rO zu5thINx=D2UA#R-bM`JO-cpTGRH2u?6-9KCg2`z5TMorLl8S>%bwKUa+PJdB@_Bb=bmA}39$MqD8W|-;$Mdwlk zlhlRf7IhN-_gOU2E)^j`6W{p~SSU+&fqa7T5DZw&)A*Xoy0RrGtmOz?D~hGfFP3dK zr*-gVK(YtFvgjfiaY})O56tn4Kxg8miJh?`36ZJe3w2)LEZ=UHWQgh11bTEOmYpyo z-Y<7Nsd<@RNSZS&&Mv^Wz{U0IP(hA!}5$u zHzkL~%W&cK4u1gofJI}eR)4ZG%^$~?7LTPu{mCg#y`Eob0RRR7000A;?M?mw literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/interp_omp.sz b/tools/test/streams/3.3.2/interp_omp.sz new file mode 100644 index 0000000000000000000000000000000000000000..0e0b968da99109c6ed874d76c6b09c4a6f320140 GIT binary patch literal 4843 zcmWgm?DUy|iJ5ta5Ca&n07+$LHiaez1`db_6T_dCA20bP1=z0u3NSLT0ZC@MDQN;Z z2ncY|DNt8xY}Nmpkiy=o%+OYM!`dO{yv*#@%m;iHJHN{d@yb5*v%D3$V&c|)nt>V2 z4B2wo+?Tz62RwD>ym-LxfKk`0y#7sIUZ>`A_V+GX#jup&5<{})g{%p>#=;-{`xm_A zP4NH4Uh_qGZ|RB`?2GpOj+w~eAp29^{8Nw1KbIrypW1CJ-ux)ES=gJ+@z1DEyrEuq z-+T9uHPxyfJr_<_@7ezQ!}W{0#~(iY$9p*Z#@e@Ay?-b!S!ioftne)%hG(1I>-B3^ ztq}6@blrQ|SoVkNQtm%YX1|#C1em^S6{4b%0K(>>zma@b=$r-{t5geC;5NT zOe?2%KU}5$D|X30sr@6c>BsXQhot$X{u%!hH%PirvETmaY>S0StlbW>@{b(v{FSw_ zIsTV-zT@HO8HZZ$pZn$>5ItMK-0fRpY+U^Q2ebQA&z;u>uW30zu={S_?oj>0*7qV? z)H9si8~j^NY?-Habiswy0WoQ2#=DhswR*bu1>QNNvHi%Kt5c6}+dtRWqm+POa6@9+EsD7_a{D$RLmskUO+HP;jlass= zyVF(o&Kkz%G{%L-W|@BjMX` z&)vo4HPJh1;mPR&-d0yXlxpt1llk|=8-}A9JA7YHD`lKs$1uI_ipgdm+EsnUTl>}0 zQ1R}gNo&JahjPZGgg;t-SIuM-Q;NWK&NkbVyIt*U^wY8|)Y&byUr*3-oD!>l;qHPh zP0D>@Tuw8MkF9i8m7QEtm)o1ZgD>JmYJ>Hm+Q}hDnuRhXr#{-l6&QWkDRL4;^G7M9!t2j zN|9^k+sB=jpVaSLzq`L=rGMq`xfg`xL@V~#PONg%KX>55o50SS-2c~TK2qUXbW>T@ z`%Iq0tCUQ!%uUn!PWa@k-n9H?p}~!ctlNf%&K`WpYufO3MU;*IyK^${3U~gs3k;}F zyYzlSF>CRiyEoq&ubzMFLhwOVwwD#c*W46&!jDTHcARr2O_FizMwQ^&liT+eMLMWQ zU3z2xLf)>LXYLL`k>Ew%Ors^xXbCi00*#hHqb1O28)&o*G};ClZ3B(=!bW>xqrI@v ZUKl-lVam)L4ICT{3ZStf1_rESRsce*8j1h_ literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/lorenzo_reg.sz b/tools/test/streams/3.3.2/lorenzo_reg.sz new file mode 100644 index 0000000000000000000000000000000000000000..ca880983648a3a4243299ac554de81881643e7d6 GIT binary patch literal 2010 zcmV<02POCr^Fs3g0s{l52LJ#70002T4FCWD0000ewJ-f(#|*7J0J>rb7B3(qYYx}- znRWq+K+lOY?~tmevU0ETre%8dJ1|;0cqvdiPcOY_AM3I#SwSt$nl0xIQTVw zhKv~R;{RIwD^4+ry%TzQ-+hU?f5e9X!2#|yxV8P-Tc|LF2@=vC!eqhs{;Jgx78@9JJn}mSV`hi+r5|Ar>0Y?Q=Z$TG;HpurZ&q4W&0GJBp?JyII1S% zAMo_2PoWC@)KHbDk_bZQse?}q8$lfs)X^e5%>)3>tib#T5Kr z2sZHb)(;r#ud=#&>sNU~U1-RDwtOfi508qWuNB!hPolb=M4iE>S z1O;YFLS%({;%yN8SkqI<6r4=b?3Hod5+q;&7Csvu|s841yQ zZy&u52ig?xy_}niHm7gY+D8AK# z@K0!Kci-x@Y^~e38csYF+P9I_%y-H+^yKmPD<6u#ioxZ3y^4;c9Otzteu7BFy(_a^NAxfdbV!Nb38 z`0xRIY;tVMUO3LX;HIsSK0uLqCEt`@Ei)~(lHM5A+{i>ATh-A*SufPdl4t>?d#@8U zPcS4NCc5PKg>y~Q#WH0Sn?`2KoovDcEQbyj^?n4~--j zn!VN>N}^_?JT3hpl(ev+<3hyV%~cv z@kD?mD#hrmQGhQ)rT1P@(kYSDAhV8b0MupFCW?5xoqH}Xj@$7kSW#*r)`UcqmVooT?t}r4I_P;IRyx{vSwxPy@W~V zH0KW8do3-m8?({N(kp0N>1A;5y_7RTdhZ1Vz4y*lj+IG;z&sLWtn5dl7zhO7VCHR2 zC8D$>I|C8~5v3*BSJa!tEh{HKH!3UL<69P`Rb%I21ur6wiMGW3`GG+qDT&9#$fkSm z^w6FlnrDXs^_;0#gLrQV%1Jl!a(QL70;2a`wFqHiT(sm-a#7xma!R^laM=fQd?+O@ zvN2K{q5S!&ge7{71IO1;Z0HUc#tjvgP3KZ&4Od(TAXj#7v2Dm8pYw<-GSIakS|w!uoK$jWI+ok62y+hL&E6 zrH?f#KMi3XT2VE{tuv782IywI|b^gX1X_46{%g9aEE4I@`=7$=(YJ2$CmZ-K~t0 z#gl7*C1#~dmrXiCHUTdJFiY4g#5+lvhY}_15g*kqNnlxIq-9iyOt!g#f_HJx5_S`( zBdn&2rVbjXN5gw>TL!EnO?Rkd@5MR^d-N!Kjmr~|gfT&1T%Ld_7JX$xI0(|A$kKwt zL6FXk>T!yeMQPPY1_x$k>#EYstW;yIt4h-@lFl1B_g)nY95~1nK`c`nDq4_yZKMQ& z1*owg#*kx7MnpuKB$W;k1rj+2mE8dn!1XK*E9A^ji6jLj1vM&-0btaej@ph%D2=L! z%9mksMzjgAIm6B|G}p0h*?LznQxaLd&w40kJwIy!Gs?m2^9!1>;SH%*7hD@PS;ojH z+Bk~uRsK_T#N{mOrvckit{hek=)vTc#>#$?$Sb!JFM(uKAv3a`(0a-g$t8>g+FVYnQ^av1 z4drF1%JO87xoD$Ii>U4ezo|vM&{sq{|G8X(c+9xaQULi^ zLQ`~LH3ubV@( zxEeSbVGKMR9nRX!026IBs>WTkC^p!j5C$OCV4$=aAnY8}z!_pjK!aC@#u%J{z z|2glcxLgdhzpFYANjFmv4`OH(O*KcQ65(*@Mubit2YqQuWkRNKi=Q&qTuL;W43_#1 ztQgW{7>J=#yn{I76UtZvfyww3e>t~NaU?nq)qTm;#Ie}P@rBwTa=d|9&uTUet3+@H s7Iix`fDt1D2!IF(03ZMW00000008`{@zPCVP(Ppm000311^@s61OElCd;kCd literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/lorenzo_reg_omp.sz b/tools/test/streams/3.3.2/lorenzo_reg_omp.sz new file mode 100644 index 0000000000000000000000000000000000000000..a31113abafafbef2b40a240304e8e78e7859a800 GIT binary patch literal 4735 zcmWgm?DUy|iJ93)hye^(fTS`rn?e%<0|!Kef#J`}kC%Lt0_;}+1sECFfFv{Bl%#_k z1O&L~CU%It##a5m2~BKkl^ERiU$=5t;+L)e=f&0UIhU5zUH`48<5KW+{_!bxEDW6r zYqxy=!@K+G?N{6Wyl38Fv#OC})18#>cYHlGx>DYYtYKKoaEd|u!#umL_0@jwe0?6> zy>%n*?Rlp^?bd&0L`bxM|HUJ*?~wNdw~80h^S|;+=o|(`+bj!u7^SG`i^yt;4SmkEk5rmy<-YS;JIomcJmPx&sye<*(Gn(+Or z{_gsE;^lhhd9M~P`S)_>tOfIao_c!Xng8AfhWB3>=AGLLc^Y?r__PuQV%MWu;ZdSi@+cxZVu1NN^hg0L`zqz#Klc(!W zft37%Z#uSf&XcT-ZoQdmtDsq$xLk3i59=KuU33Tt|Q<~;e4xTjm(FomP;yXLWzAMS;zIA-_@Z_m8*Z#dW&8gtFZ#>l6f9|Wmy@Ska^){>z`NLY*a5dQ7 zR-|FBkFOsi&z=&6lY+|jPa16!`yJ)>KQ8_8xA*<*V?BH0-$y^$$Wi%jio4Tcm+~G3 z_Z4$ydoSyIymz`+%>3&2@-pIkFAhpcdKC0>=8<&eV=Ln#Z{8^3I?)radd&62;Z;jd?0VVh z@1T|y@@sW*=-s-C-4EAxZdWzg^E4rT53AFU)7sY#Mems2A6Hs!_cy5TXjOIU{+8NF zhoXi23(_5Yw_OiajTaH z37l=pc(tg#%-SWxwNd5tjyZ)2u1C|RHf_}W_@E|C^eJOd6PJTlZ+QR5b9W2=6eT~M zWohR4{muc?sqgmY*-voM)k@m=-Kntofa!^z$;O7U2X5SKT;+76|MGP2J2alfaka)5FZ7o+WCfjXrGcrH#}2 z*iFucEKuk@p=N95ut`mI;-$UjT2A6F$(OBllJjTPR#v^+Te87L^4J3B;yu2aObrby z1eZD;aWqj*T(U&NlYIe`)+R4@0gI(wTRbmG9GK7%+&@43&WW^!b1~v-N3>rHJTQMH z`-^Q#gHZfFwMq~96R&f?Vlsoart984S_jzmp8x18kIJc7zh@g?jiKqpZFgrrU>z-HMvIxz zVrI0M87*c;TQj4rnbFqFXlrJ)(=^&?8tpWVcAD^Xnv|J28aOx@6d)rsSjUk7#zhN^ literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/lorenzo_reg_rel1e-5.sz b/tools/test/streams/3.3.2/lorenzo_reg_rel1e-5.sz new file mode 100644 index 0000000000000000000000000000000000000000..fdc488baf99b80e3503be9f690ce7e1ddc7fa09f GIT binary patch literal 3332 zcmV+f4g2yC^Fs3g0s{ll3;+NC0001|7XSbN0000ewJ-f(rxyKX0IH_oBt8%@k2MER zNIEteV1XitA>m%6<6s^52#I_S#tZ^g-Y?ttb_uY6==;2&{{Ws{od5c^2>#C;m#-9| z+L%lM)Bykjjsk!~@M{^&WWLH)zVZ#|9cBHU>|6>1NKW#S`K~Ln0v}`{51D%&%iZH2 zUw{CA01q&ZbIgEojJ0pB_N%zXY@^u3U*aD-#c(ge>#q-U4fc0k%9Ijq0MQLQP9yLN!y%NC2RQiSSZEs9h?T76&!!RV)o^m6}1tFcqajHN9FvT^i|C zqy00-{Bq05KLr&2L`zSI{%Gh$*zv|0UwjZexP$*B-e*KTf1VBg4939lFZ>DTT)}dt z;0ca1oHlEYU^If8xr~@U%zyd2{O!P7e!sJf<@=`cl;4B$^U0rglbI;N`)<3Bya16C zcm$>!h>vxAzy>@4-}r4@7-OueRWk;)fG<`<^yaz#K?z1y;qpc=6u5M!z6#f_9e_d8}m-&0oas=MA;w_fCU+A%UCyW*G=&y=N-gyP+olYJ830UPk`ju@- zfL}c*Vb2Jse?~b)Wc}*c_Un|KfAU!L&#qq!In|G1_J&P!-=_rbfakly^mEqcJG~yD zk5kHtp-2g@MR2FmYD+5o8zwfCpN3A7q<8Zb_i&u93*nz?cVXb$5}|;6C)6VuL-O?d zXIye1%VSXNs5AS3O0pgt&p5AG6_k3l0Z677g;}?m3Ug1;AKaKon zK1P%Os*f1d z^DEwt{q=cg^-vOpK;t5T7CH><=-@D65d~{@&{4dA3k4%k2j+~w!gwV-59hFVU z%BJ4m63UR=T`6;B4*df0>~7sx;)fOuG|9L{neJ}m9Yj)LSXdda2~L~^?(9M@0T~BH z)%dgHkc8dU$Y@%C+u|4lPS2hqCY+!^7}%8rz{>u-=$S!xhU~@_m;`E?NHcdA37%4lNgdAJ zUCHQj)KH@(Y0qm<&+dAl;@83L3D)i!f>F_jbd><+ZlE59ZH!!%_EZ}7v2EG&aH-Ux z+})`aA0+!gt%)RuVD7F7J5z@k^70hzE|xcChMG%Mxw~K}*WAjSIj6g8bIxO-DPBq| zM?=gGt`9quYhn#2cZ5SA0(m|4;9;i#1~Z6hYOv@8e6NfQ}RotLcO!zQh1_qFZG_DZ^AdA8{shs7eDNds*b4Xl4VnHeb}LtBb2$BhFR9&kolmaV@=ZJZuD~HESq;ynQgMPSS4gK zp_gJHr#F^%jIoLy8DTI44&uCJ8G|Q~(NHp)iRE!>%#Oj;G1)(>#LcDNU|b=%UBSsf z#@#0ZrcyD*MMA=C^GLW{ EEHmH?mLKWs3m7Fdjh(RY_AVrlsmbqzeMA=~>s*;Cb z)F;K~7B;;hXP}4F#4N6i!O4}gY!+HxxpS0+tNS9!1Ff(T@Xw{72D``gPzOu7ExoKP zK3mWr;sNcVoMdzh;=D^t`ntg-NpLJj4Iu%bse{L4cLld`%Az>>!u-CZm3r%;}_Lc3eRmLPswYgT$p%BWjlDMXY_p-5>{Rd<9XqJ0yyYg52!oD>iU zV#+9HsVNzBWyb;sG%(D%AYg$4QAmPPr0DWxW3kQ|7;OZ-=;WlZ=tNZpHxpjcRQV&I z0~tFwxw|GFF|f6h)9wo>CprVgh~~i2NEDON))7w84(c?Nm5Wy zs55K_xq%^iaHf&$^(aGr@r!tb{C6fiO<&U5>27N#Ei}4iQNSKZL4%$`rul)1}fBi8l{bFfWxID}Q%m!~L zg8#;AoA35^a9nGam2_qO<)O3t!(+@D<#lJzhrj)Q5TDldrZtidQRH3+YR=Hx@975- zK{4acYeGTKm8HnLd)#u%USFGql2-gG}HzgenXy zjO|Z7*O<=0GE_Rmp724v2B=q}tQgS`WU!VIBLfJ42nYZm0000000000px|Of4PXgB Opa1{>0RRR70009m4JY3K literal 0 HcmV?d00001 diff --git a/tools/test/streams/3.3.2/lossless.sz b/tools/test/streams/3.3.2/lossless.sz new file mode 100644 index 0000000000000000000000000000000000000000..82fc2f06e5f777e90a3801f3537a325a99b5fb50 GIT binary patch literal 735 zcmXAneJs>*0LFhgE}B}qlvk<^yX^JPWS7QF>9!t;&4vs&B!Ta*Gi;$w&(fmdH#OpkA;pgP7=u@CuT;v8TeNR z|BYwPw0khlUT4<`Sh-3-gO30oUjaEQrh9+@=>`GT%>s&p1t=B^SnuGmq??ESC68Nm zJR)m(&>G&agvV2X$GX)#qQ7#uP|o4BmP2e92Y(rt-J}9CRJi#w9oZxu^Ae6?S;i5J zXB3S z4n?XQnkRAuut--3@N2$}7jxfZ^3zfhRJ)J#-8fG!n~KO-pnw~(UH4T`x4IkdXFfR^^; zQ}wn~8khH)g#OhL7e+^zY5!;Mb{)yhxkjYQ<#fT|4LVU(OkX|EqZv8r)ZILr9yzHX zf1b&Rw#}8emAa6zBokEcYM|<=LaF=XgpUs(MwR1RdntCc+`%PT5gOwPasEUBnCN`n$i2G$DYLY7 zr}Cggwt2^gsq3B-pS*3E)`}|u&6=(Xt6tPtWAgN3X1SGP4+QSFaIE%wqC?NBEfHFq zI9SqD$%<_ztKKGdmRUL5fHBrHrg^9`)?mxD8;^S0G~eV7PJLgVV&T-^dzD&NGte8O zHQGDM!j$4rhv^|J+PimKY{s{y!k`B?yJ8G_QQM`m1Q{*EzYL~Hg*UsB z>HuU!8el;D0{}j7J@kMFQhq$iJ%f%-KNMaUe(!=Vui5q=d<8CK-LYj`Kc$h$y|=Wg3%XW~9&-PWmOZ(>*4MAm5ra^LQFcJW)ZugdYRr1Q-~TYUR^ z|8C*w`<0yjmE(SWyX2pmP5jv(%0uHB?-=75zlDp&SMi32PxvPWtt)E0-|z9A_jW%= z_s4}0arbWTf$h~EEseuJrr|OCm4WQT-)nsZ>Qi_LRaQS#)HA)(v`OlZX7ELyK!G$( zYqaUISub0-^l+`CTUP0>QC9TBsPug{SE%($z0Q>lbN%^Bc${ndkCO=I{Pfk zviE$`c}IDd@9DSf)t_nXn@y(gK5>1U=0Eaop!3B$seu13()i8TXPw-aZs`iB z4O&uD4K87hVhSTE9gn~$uPz~PG>vhCi3wD-D?;RTh9(D=Iwh8H?)Y$EVWE5*;)y3q||7q#qJ8h#8A7W%EKSoQ^yFa&~{Q`YjUs=a!ASaq$dMAtYG zgIJ;l7gl)}O#tA_q`Mna5G<3k^@msQZ6i%bdGtW;IUEI@s#h-RBXr +# tools/test/sz3LinkTokens.sh --self-test +# +# Not "the tokens that contain sz3". On a GitHub runner the checkout is at +# /home/runner/work/SZ3/SZ3, because the repository is named SZ3, so the name of the project is +# in every absolute path on the line and -Wl,-rpath-link,/lib matches it while linking +# nothing. A token links an SZ3 library if it is -l, or a path whose *file +# name* is an SZ3 library. -L, -I and -Wl,... name places to look, not libraries to link. +set -u + +sz3_link_tokens() { + tr '[:space:]' '\n' | awk ' + { t = $0; gsub(/^"+|"+$/, "", t) } + t == "" { next } + t ~ /^-l/ { if (tolower(t) ~ /sz3/) print t; next } + t ~ /^-/ { next } + { n = tolower(t); sub(/.*\//, "", n) + if (n ~ /sz3/ && n ~ /\.(so|a|dylib|dll|lib)(\.[0-9]+)*$/) print t }' +} + +# Every check that reads this file is only as good as the line above, so prove it both ways. +# The tokens are real ones, off the link lines in the GROMACS workflow's own runs. +self_test() { + bad=0 + while IFS='|' read -r tok want; do + [ -n "$tok" ] || continue + got=no + [ -n "$(printf '%s\n' "$tok" | sz3_link_tokens)" ] && got=yes + if [ "$got" != "$want" ]; then + echo "sz3LinkTokens self-test: '$tok' -> $got, expected $want" + bad=1 + fi + done <<'CASES' +/home/runner/work/SZ3/SZ3/_prefix/lib/libhdf5sz3.so|yes +lib/libhdf5sz3.so|yes +lib/libhdf5sz3.so.3.3.3|yes +libSZ3c.a|yes +-lhdf5sz3|yes +-l:libhdf5sz3.so|yes +-Wl,-rpath-link,/home/runner/work/SZ3/SZ3/_m/found/b-OFF/lib|no +-Wl,-rpath,"$ORIGIN/../lib:/home/runner/work/SZ3/SZ3/_prefix/lib"|no +-L/home/runner/work/SZ3/SZ3/_prefix/lib|no +-I/home/runner/work/SZ3/SZ3/include|no +/home/runner/work/SZ3/SZ3/gbuild/lib/libgromacs.so.10|no +/home/runner/work/SZ3/SZ3/_m/found/b-OFF/lib|no +"CMakeFiles/h5md-test.dir/h5md.cpp.o"|no +/home/runner/micromamba/envs/h5/lib/libhdf5.so|no +CASES + [ "$bad" = 0 ] || return 1 + echo "sz3LinkTokens self-test: ok" +} + +case "${1:-}" in +--self-test) + self_test + ;; +"" | -h | --help) + sed -n '2,12p' "$0" + exit 1 + ;; +*) + [ -f "$1" ] || { echo "no such link line: $1" >&2; exit 1; } + sz3_link_tokens < "$1" + ;; +esac diff --git a/tools/zstd/CMakeLists.txt b/tools/zstd/CMakeLists.txt new file mode 100644 index 00000000..9d5eca11 --- /dev/null +++ b/tools/zstd/CMakeLists.txt @@ -0,0 +1,73 @@ +# Vendored Zstd, unmodified. Do not edit lib/; a re-vendor replaces it wholesale. +# +# Upstream: https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz +# SHA256: 8c29e06cf42aacc1eafc4077ae2ec6c6fcb96a626157e0593d5e82a34fd403c1 +# License: LICENSE beside this file (BSD-3-Clause; upstream dual-licenses GPLv2) +# +# Not 1.5.7: it made level 3, the one Lossless_zstd uses, 26-30% slower for under 0.6% of size. +# Its correctness fix needs a ZSTD_CCtx reused across calls; SZ3 makes one per ZSTD_compress. + +set(_zstd_lib "${CMAKE_CURRENT_SOURCE_DIR}/lib") + +# Listed rather than globbed: a glob would silently start compiling whatever a re-vendor drops in. +add_library(sz3_zstd STATIC + "${_zstd_lib}/common/debug.c" + "${_zstd_lib}/common/entropy_common.c" + "${_zstd_lib}/common/error_private.c" + "${_zstd_lib}/common/fse_decompress.c" + "${_zstd_lib}/common/pool.c" + "${_zstd_lib}/common/threading.c" + "${_zstd_lib}/common/xxhash.c" + "${_zstd_lib}/common/zstd_common.c" + + "${_zstd_lib}/compress/fse_compress.c" + "${_zstd_lib}/compress/hist.c" + "${_zstd_lib}/compress/huf_compress.c" + "${_zstd_lib}/compress/zstd_compress.c" + "${_zstd_lib}/compress/zstd_compress_literals.c" + "${_zstd_lib}/compress/zstd_compress_sequences.c" + "${_zstd_lib}/compress/zstd_compress_superblock.c" + "${_zstd_lib}/compress/zstd_double_fast.c" + "${_zstd_lib}/compress/zstd_fast.c" + "${_zstd_lib}/compress/zstd_lazy.c" + "${_zstd_lib}/compress/zstd_ldm.c" + "${_zstd_lib}/compress/zstd_opt.c" + "${_zstd_lib}/compress/zstdmt_compress.c" + + "${_zstd_lib}/decompress/huf_decompress.c" + "${_zstd_lib}/decompress/zstd_ddict.c" + "${_zstd_lib}/decompress/zstd_decompress.c" + "${_zstd_lib}/decompress/zstd_decompress_block.c" +) + +# sz3_zstd, not zstd: a project that adds SZ3 as a subdirectory may have a zstd target of its own, +# and libzstd would overwrite the one already in the install prefix. +# Hidden visibility keeps ZSTD_* out of libhdf5sz3.so, where it interposes on the consumer's Zstd. +set_target_properties(sz3_zstd PROPERTIES + EXPORT_NAME zstd + POSITION_INDEPENDENT_CODE ON + C_VISIBILITY_PRESET hidden) + +# ZSTDLIB_VISIBILITY=: Zstd's headers mark its API visibility("default"), past C_VISIBILITY_PRESET. +# ZSTD_DISABLE_ASM=1: huf_decompress_amd64.S is not in the source list above. +# ZSTD_LEGACY_SUPPORT=0: lib/legacy is not vendored. +target_compile_definitions(sz3_zstd + PRIVATE ZSTDLIB_VISIBILITY= ZSTDERRORLIB_VISIBILITY= ZSTD_DISABLE_ASM=1 ZSTD_LEGACY_SUPPORT=0) + +# PRIVATE only: a public include directory would put this zstd.h ahead of a consumer's own. +target_include_directories(sz3_zstd + PRIVATE + "${_zstd_lib}" + "${_zstd_lib}/common" + "${_zstd_lib}/compress" + "${_zstd_lib}/decompress" +) + +if (SZ3_INSTALL) + install(TARGETS sz3_zstd EXPORT SZ3Targets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) +endif () + +export(TARGETS sz3_zstd FILE zstd.cmake) diff --git a/tools/zstd/COPYING b/tools/zstd/COPYING new file mode 100644 index 00000000..ecbc0593 --- /dev/null +++ b/tools/zstd/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/tools/zstd/LICENSE b/tools/zstd/LICENSE new file mode 100644 index 00000000..75800288 --- /dev/null +++ b/tools/zstd/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/zstd/lib/common/allocations.h b/tools/zstd/lib/common/allocations.h new file mode 100644 index 00000000..5e899550 --- /dev/null +++ b/tools/zstd/lib/common/allocations.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "compiler.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ diff --git a/tools/zstd/lib/common/bits.h b/tools/zstd/lib/common/bits.h new file mode 100644 index 00000000..def56c47 --- /dev/null +++ b/tools/zstd/lib/common/bits.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ diff --git a/tools/zstd/lib/common/bitstream.h b/tools/zstd/lib/common/bitstream.h new file mode 100644 index 00000000..67604498 --- /dev/null +++ b/tools/zstd/lib/common/bitstream.h @@ -0,0 +1,457 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/*========================================= +* Target specific +=========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ +# elif defined(__ICCARM__) +# include +# endif +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef size_t BitContainerType; +typedef struct { + BitContainerType bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + return BIT_reloadDStream_internal(bitD); +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not never beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ + return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStream_internal(bitD); + } + if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr => cautious update */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/tools/zstd/lib/common/compiler.h b/tools/zstd/lib/common/compiler.h new file mode 100644 index 00000000..31880ecb --- /dev/null +++ b/tools/zstd/lib/common/compiler.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +#include + +#include "portability_macros.h" + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explicitly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE FORCE_INLINE_TEMPLATE +#endif + +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ +#if defined(__GNUC__) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. + */ +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) +# else +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) +# endif +# else +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# endif +#endif + +/** + * Helper function to perform a wrapped pointer difference without trigging + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/tools/zstd/lib/common/cpu.h b/tools/zstd/lib/common/cpu.h new file mode 100644 index 00000000..0e684d9a --- /dev/null +++ b/tools/zstd/lib/common/cpu.h @@ -0,0 +1,249 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(__clang__) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } +#endif +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/tools/zstd/lib/common/debug.c b/tools/zstd/lib/common/debug.c new file mode 100644 index 00000000..9d0b7d22 --- /dev/null +++ b/tools/zstd/lib/common/debug.c @@ -0,0 +1,30 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) +/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a + * translation unit is empty. So remove this from Linux kernel builds, but + * otherwise just leave it in. + */ +int g_debuglevel = DEBUGLEVEL; +#endif diff --git a/tools/zstd/lib/common/debug.h b/tools/zstd/lib/common/debug.h new file mode 100644 index 00000000..a16b69e5 --- /dev/null +++ b/tools/zstd/lib/common/debug.h @@ -0,0 +1,116 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } \ + } while (0) + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LINE_AS_STRING TOSTRING(__LINE__) + +# define DEBUGLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } \ + } while (0) +#else +# define RAWLOG(l, ...) do { } while (0) /* disabled */ +# define DEBUGLOG(l, ...) do { } while (0) /* disabled */ +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/tools/zstd/lib/common/entropy_common.c b/tools/zstd/lib/common/entropy_common.c new file mode 100644 index 00000000..e2173afb --- /dev/null +++ b/tools/zstd/lib/common/entropy_common.c @@ -0,0 +1,340 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#include "huf.h" +#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = ZSTD_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int flags) +{ +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)flags; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/tools/zstd/lib/common/error_private.c b/tools/zstd/lib/common/error_private.c new file mode 100644 index 00000000..075fc5ef --- /dev/null +++ b/tools/zstd/lib/common/error_private.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Data corruption detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/tools/zstd/lib/common/error_private.h b/tools/zstd/lib/common/error_private.h new file mode 100644 index 00000000..0156010c --- /dev/null +++ b/tools/zstd/lib/common/error_private.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include "../zstd_errors.h" /* enum list */ +#include "compiler.h" +#include "debug.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/tools/zstd/lib/common/fse.h b/tools/zstd/lib/common/fse.h new file mode 100644 index 00000000..2ae128e6 --- /dev/null +++ b/tools/zstd/lib/common/fse.h @@ -0,0 +1,640 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, (size_t)statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, (size_t)statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/tools/zstd/lib/common/fse_decompress.c b/tools/zstd/lib/common/fse_decompress.c new file mode 100644 index 00000000..0dcc4640 --- /dev/null +++ b/tools/zstd/lib/common/fse_decompress.c @@ -0,0 +1,313 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#include "zstd_deps.h" /* ZSTD_memcpy */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + assert(op >= ostart); + return (size_t)(op-ostart); +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; + + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); + + /* normal FSE decoding mode */ + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/tools/zstd/lib/common/huf.h b/tools/zstd/lib/common/huf.h new file mode 100644 index 00000000..99bf85d6 --- /dev/null +++ b/tools/zstd/lib/common/huf.h @@ -0,0 +1,286 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ + +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; + + +/* **************************************** + * HUF detailed API + * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; + +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int flags); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#endif +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif + +#endif /* HUF_H_298734234 */ + +#if defined (__cplusplus) +} +#endif diff --git a/tools/zstd/lib/common/mem.h b/tools/zstd/lib/common/mem.h new file mode 100644 index 00000000..096f4be5 --- /dev/null +++ b/tools/zstd/lib/common/mem.h @@ -0,0 +1,426 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include +# else +# include /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * Default : method 1 if supported, else method 0 + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# ifdef __GNUC__ +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return MEM_swap32_fallback(in); +#endif +} + +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/tools/zstd/lib/common/pool.c b/tools/zstd/lib/common/pool.c new file mode 100644 index 00000000..3adcefc9 --- /dev/null +++ b/tools/zstd/lib/common/pool.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ +#include "zstd_deps.h" /* size_t */ +#include "debug.h" /* assert */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(ZSTD_pthread_t)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} + + +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job job; + job.function = function; + job.opaque = opaque; + assert(ctx != NULL); + if (ctx->shutdown) return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/tools/zstd/lib/common/pool.h b/tools/zstd/lib/common/pool.h new file mode 100644 index 00000000..cca4de73 --- /dev/null +++ b/tools/zstd/lib/common/pool.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include "zstd_deps.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "../zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and reuse existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(const POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/tools/zstd/lib/common/portability_macros.h b/tools/zstd/lib/common/portability_macros.h new file mode 100644 index 00000000..e50314a7 --- /dev/null +++ b/tools/zstd/lib/common/portability_macros.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#elif defined(__APPLE__) +# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/tools/zstd/lib/common/threading.c b/tools/zstd/lib/common/threading.c new file mode 100644 index 00000000..25bb8b98 --- /dev/null +++ b/tools/zstd/lib/common/threading.c @@ -0,0 +1,182 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +#include "threading.h" + +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ + + +/* === Dependencies === */ +#include +#include + + +/* === Implementation === */ + +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + +static unsigned __stdcall worker(void *arg) +{ + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + ZSTD_thread_params_t thread_param; + (void)unused; + + if (thread==NULL) return -1; + *thread = NULL; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (*thread==NULL) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return errno; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread) +{ + DWORD result; + + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + + switch (result) { + case WAIT_OBJECT_0: + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + assert(mutex != NULL); + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + assert(mutex != NULL); + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + ZSTD_free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + assert(cond != NULL); + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + assert(cond != NULL); + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + ZSTD_free(*cond); + return ret; + } +} + +#endif diff --git a/tools/zstd/lib/common/threading.h b/tools/zstd/lib/common/threading.h new file mode 100644 index 00000000..fb5c1c87 --- /dev/null +++ b/tools/zstd/lib/common/threading.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#include "debug.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef HANDLE ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#if DEBUGLEVEL < 1 + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#endif + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/tools/zstd/lib/common/xxhash.c b/tools/zstd/lib/common/xxhash.c new file mode 100644 index 00000000..052cd522 --- /dev/null +++ b/tools/zstd/lib/common/xxhash.c @@ -0,0 +1,18 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/tools/zstd/lib/common/xxhash.h b/tools/zstd/lib/common/xxhash.h new file mode 100644 index 00000000..e59e4426 --- /dev/null +++ b/tools/zstd/lib/common/xxhash.h @@ -0,0 +1,7020 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Local adaptations for Zstandard */ + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n +#include +#include + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/tools/zstd/lib/common/zstd_internal.h b/tools/zstd/lib/common/zstd_internal.h new file mode 100644 index 00000000..ecb9cfba --- /dev/null +++ b/tools/zstd/lib/common/zstd_internal.h @@ -0,0 +1,392 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "cpu.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: zstd_decompress_block, fullbench */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/tools/zstd/lib/common/zstd_trace.h b/tools/zstd/lib/common/zstd_trace.h new file mode 100644 index 00000000..da20534e --- /dev/null +++ b/tools/zstd/lib/common/zstd_trace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386 and aarch64 + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + unsigned streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + unsigned dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_TRACE_H */ diff --git a/tools/zstd/lib/compress/clevels.h b/tools/zstd/lib/compress/clevels.h new file mode 100644 index 00000000..c18da465 --- /dev/null +++ b/tools/zstd/lib/compress/clevels.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CLEVELS_H +#define ZSTD_CLEVELS_H + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "../zstd.h" + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 + +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */ + { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + + + +#endif /* ZSTD_CLEVELS_H */ diff --git a/tools/zstd/lib/compress/fse_compress.c b/tools/zstd/lib/compress/fse_compress.c new file mode 100644 index 00000000..1ce3cf16 --- /dev/null +++ b/tools/zstd/lib/compress/fse_compress.c @@ -0,0 +1,625 @@ +/* ****************************************************************** + * FSE : Finite State Entropy encoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/compiler.h" +#include "../common/mem.h" /* U32, U16, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "hist.h" /* HIST_count_wksp */ +#include "../common/bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "../common/zstd_deps.h" /* ZSTD_memset */ +#include "../common/bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 const maxSV1 = maxSymbolValue+1; + + U16* cumul = (U16*)workSpace; /* size = maxSV1 */ + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ + + U32 highThreshold = tableSize-1; + + assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ + + /* For explanations on how to distribute symbol values over the table : + * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u <= maxSV1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + assert(normalizedCounter[u-1] >= 0); + cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; + assert(cumul[u] >= cumul[u-1]); /* no overflow */ + } } + cumul[maxSV1] = (U16)(tableSize+1); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + /* Case for no low prob count symbols. Lay down 8 bytes at a time + * to reduce branch misses since we are operating on a small block + */ + BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Spread symbols across the table. Lack of lowprob symbols means that + * we don't need variable sized inner loop, so we can unroll the loop and + * reduce branch misses. + */ + { size_t position = 0; + size_t s; + size_t const unroll = 2; /* Experimentally determined optimal unroll */ + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableSymbol[uPosition] = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); /* Must have initialized all positions */ + } + } else { + U32 position = 0; + U32 symbol; + for (symbol=0; symbol highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } } + assert(position==0); /* Must have initialized all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u 1); + { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); + U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); + total += (unsigned)normalizedCounter[s]; + } } } } + +#if 0 /* debug : symbol costs */ + DEBUGLOG(5, "\n --- table statistics : "); + { U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + symbol, normalizedCounter[symbol], + FSE_getMaxNbBits(symbolTT, symbol), + (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + } } +#endif + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** +* FSE NCount encoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog + + 4 /* bitCount initialized at 4 */ + + 2 /* first two symbols may use one additional bit each */) / 8) + + 1 /* round up to whole nb bytes */ + + 2 /* additional two bytes for bitstream flush */; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; + + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = (int)tableLog+1; + + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ + while (symbol >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (symbol >= start+3) { + start+=3; + bitStream += 3U << bitCount; + bitCount += 2; + } + bitStream += (symbol-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += (U32)count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; } + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + assert(out >= ostart); + return (size_t)(out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); +} + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = lowProbCount; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) { maxV=s; maxC=count[s]; } + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) { ToDistribute--; norm[s]++; } + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue, unsigned useLowProbCount) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; + U64 const scale = 62 - tableLog; + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = lowProbCount; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) { largestP=proba; largest=s; } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U< FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/tools/zstd/lib/compress/hist.c b/tools/zstd/lib/compress/hist.c new file mode 100644 index 00000000..e2fb431f --- /dev/null +++ b/tools/zstd/lib/compress/hist.c @@ -0,0 +1,181 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/mem.h" /* U32, BYTE, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "../common/error_private.h" /* ERROR */ +#include "hist.h" + + +/* --- Error management --- */ +unsigned HIST_isError(size_t code) { return ERR_isError(code); } + +/*-************************************************************** + * Histogram functions + ****************************************************************/ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned largestCount=0; + + ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip largestCount) largestCount = count[s]; + } + + return largestCount; +} + +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; + +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ +static size_t HIST_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + HIST_checkInput_e check, + U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + /* safety checks */ + assert(*maxSymbolValuePtr <= 255); + if (!sourceSize) { + ZSTD_memset(count, 0, countSize); + *maxSymbolValuePtr = 0; + return 0; + } + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip max) max = Counting1[s]; + } } + + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } + return (size_t)max; +} + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} + +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} + +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); +} +#endif diff --git a/tools/zstd/lib/compress/hist.h b/tools/zstd/lib/compress/hist.h new file mode 100644 index 00000000..887896b8 --- /dev/null +++ b/tools/zstd/lib/compress/hist.h @@ -0,0 +1,75 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/zstd_deps.h" /* size_t */ + + +/* --- simple histogram functions --- */ + +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ + + +/* --- advanced histogram functions --- */ + +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/** HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/** HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` + */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +/** HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). + */ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); diff --git a/tools/zstd/lib/compress/huf_compress.c b/tools/zstd/lib/compress/huf_compress.c new file mode 100644 index 00000000..ea000723 --- /dev/null +++ b/tools/zstd/lib/compress/huf_compress.c @@ -0,0 +1,1464 @@ +/* ****************************************************************** + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" +#include "hist.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "../common/fse.h" /* header compression */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Required declarations +****************************************************************/ +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + + +/* ************************************************************** +* Debug Traces +****************************************************************/ + +#if DEBUGLEVEL >= 2 + +static size_t showU32(const U32* arr, size_t size) +{ + size_t u; + for (u=0; u= add) { + assert(add < align); + assert(((size_t)aligned & mask) == 0); + *workspaceSizePtr -= add; + return aligned; + } else { + *workspaceSizePtr = 0; + return NULL; + } +} + + +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 + +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; + +static size_t +HUF_compressWeights(void* dst, size_t dstSize, + const void* weightTable, size_t wtSize, + void* workspace, size_t workspaceSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); + + /* init conditions */ + if (wtSize <= 1) return 0; /* Not compressible */ + + /* Scan input and build symbol stats */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return (size_t)(op-ostart); +} + +static size_t HUF_getNbBits(HUF_CElt elt) +{ + return elt & 0xFF; +} + +static size_t HUF_getNbBitsFast(HUF_CElt elt) +{ + return elt; +} + +static size_t HUF_getValue(HUF_CElt elt) +{ + return elt & ~(size_t)0xFF; +} + +static size_t HUF_getValueFast(HUF_CElt elt) +{ + return elt; +} + +static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) +{ + assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); + *elt = nbBits; +} + +static void HUF_setValue(HUF_CElt* elt, size_t value) +{ + size_t const nbBits = HUF_getNbBits(*elt); + if (nbBits > 0) { + assert((value >> nbBits) == 0); + *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); + } +} + +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable) +{ + HUF_CTableHeader header; + ZSTD_memcpy(&header, ctable, sizeof(header)); + return header; +} + +static void HUF_writeCTableHeader(HUF_CElt* ctable, U32 tableLog, U32 maxSymbolValue) +{ + HUF_CTableHeader header; + HUF_STATIC_ASSERT(sizeof(ctable[0]) == sizeof(header)); + ZSTD_memset(&header, 0, sizeof(header)); + assert(tableLog < 256); + header.tableLog = (BYTE)tableLog; + assert(maxSymbolValue < 256); + header.maxSymbolValue = (BYTE)maxSymbolValue; + ZSTD_memcpy(ctable, &header, sizeof(header)); +} + +typedef struct { + HUF_CompressWeightsWksp wksp; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) +{ + HUF_CElt const* const ct = CTable + 1; + BYTE* op = (BYTE*)dst; + U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp)); + + assert(HUF_readCTableHeader(CTable).maxSymbolValue == maxSymbolValue); + assert(HUF_readCTableHeader(CTable).tableLog == huffLog); + + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + wksp->bitsToWeight[0] = 0; + for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; + + /* attempt weights compression by FSE */ + if (maxDstSize < 1) return ERROR(dstSize_tooSmall); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); + if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; +} + + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + HUF_CElt* const ct = CTable + 1; + + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + + *maxSymbolValuePtr = nbSymbols - 1; + + HUF_writeCTableHeader(CTable, tableLog, *maxSymbolValuePtr); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 curr = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = curr; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n HUF_readCTableHeader(CTable).maxSymbolValue) + return 0; + return (U32)HUF_getNbBits(ct[symbolValue]); +} + + +/** + * HUF_setMaxHeight(): + * Try to enforce @targetNbBits on the Huffman tree described in @huffNode. + * + * It attempts to convert all nodes with nbBits > @targetNbBits + * to employ @targetNbBits instead. Then it adjusts the tree + * so that it remains a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value (expected <= targetNbBits). + * + * @param huffNode The Huffman tree modified in place to enforce targetNbBits. + * It's presumed sorted, from most frequent to rarest symbol. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param targetNbBits The allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect targetNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment. + */ +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + /* early exit : no elt > targetNbBits, so the tree is already valid. */ + if (largestBits <= targetNbBits) return largestBits; + + DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - targetNbBits); + int n = (int)lastNonNull; + + /* Adjust any ranks > targetNbBits to targetNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ + while (huffNode[n].nbBits > targetNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)targetNbBits; + n--; + } + /* n stops at huffNode[n].nbBits <= targetNbBits */ + assert(huffNode[n].nbBits <= targetNbBits); + /* n end at index of smallest symbol using < targetNbBits */ + while (huffNode[n].nbBits == targetNbBits) --n; + + /* renorm totalCost from 2^largestBits to 2^targetNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert(((U32)totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - targetNbBits); + assert(totalCost > 0); + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = targetNbBits; + int pos; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ + rankLast[targetNbBits-currentNbBits] = (U32)pos; + } } + + while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ + U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ + totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + /* special case : no rank 1 symbol (using targetNbBits-1); + * let's create one from largest rank 0 (using targetNbBits). + */ + if (rankLast[1] == noSymbol) { + while (huffNode[n].nbBits == targetNbBits) n--; + huffNode[n+1].nbBits--; + assert(n >= 0); + rankLast[1] = (U32)(n+1); + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ + + return targetNbBits; +} + +typedef struct { + U16 base; + U16 curr; +} rankPos; + +typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; + +/* Number of buckets available for HUF_sort() */ +#define RANK_POSITION_TABLE_SIZE 192 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. + * Strategy is to use as many buckets as possible for representing distinct + * counts while using the remainder to represent all "large" counts. + * + * To satisfy this requirement for 192 buckets, we can do the following: + * Let buckets 0-166 represent distinct counts of [0, 166] + * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. + */ +#define RANK_POSITION_MAX_COUNT_LOG 32 +#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) + +/* Return the appropriate bucket index for a given count. See definition of + * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. + */ +static U32 HUF_getIndex(U32 const count) { + return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) + ? count + : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; +} + +/* Helper swap function for HUF_quickSortPartition() */ +static void HUF_swapNodes(nodeElt* a, nodeElt* b) { + nodeElt tmp = *a; + *a = *b; + *b = tmp; +} + +/* Returns 0 if the huffNode array is not sorted by descending count */ +MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { + U32 i; + for (i = 1; i < maxSymbolValue1; ++i) { + if (huffNode[i].count > huffNode[i-1].count) { + return 0; + } + } + return 1; +} + +/* Insertion sort by descending order */ +HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { + int i; + int const size = high-low+1; + huffNode += low; + for (i = 1; i < size; ++i) { + nodeElt const key = huffNode[i]; + int j = i - 1; + while (j >= 0 && huffNode[j].count < key.count) { + huffNode[j + 1] = huffNode[j]; + j--; + } + huffNode[j + 1] = key; + } +} + +/* Pivot helper function for quicksort. */ +static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { + /* Simply select rightmost element as pivot. "Better" selectors like + * median-of-three don't experimentally appear to have any benefit. + */ + U32 const pivot = arr[high].count; + int i = low - 1; + int j = low; + for ( ; j < high; j++) { + if (arr[j].count > pivot) { + i++; + HUF_swapNodes(&arr[i], &arr[j]); + } + } + HUF_swapNodes(&arr[i + 1], &arr[high]); + return i + 1; +} + +/* Classic quicksort by descending with partially iterative calls + * to reduce worst case callstack size. + */ +static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { + int const kInsertionSortThreshold = 8; + if (high - low < kInsertionSortThreshold) { + HUF_insertionSort(arr, low, high); + return; + } + while (low < high) { + int const idx = HUF_quickSortPartition(arr, low, high); + if (idx - low < high - idx) { + HUF_simpleQuickSort(arr, low, idx - 1); + low = idx + 1; + } else { + HUF_simpleQuickSort(arr, idx + 1, high); + high = idx - 1; + } + } +} + +/** + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. + * + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + */ +static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) { + U32 n; + U32 const maxSymbolValue1 = maxSymbolValue+1; + + /* Compute base and set curr to base. + * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1. + * See HUF_getIndex to see bucketing strategy. + * We attribute each symbol to lowerRank's base value, because we want to know where + * each rank begins in the output, so for rank R we want to count ranks R+1 and above. + */ + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = HUF_getIndex(count[n]); + assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); + rankPosition[lowerRank].base++; + } + + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + /* Set up the rankPosition table */ + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } + + /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ + for (n = 0; n < maxSymbolValue1; ++n) { + U32 const c = count[n]; + U32 const r = HUF_getIndex(c) + 1; + U32 const pos = rankPosition[r].curr++; + assert(pos < maxSymbolValue1); + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } + + /* Sort each bucket. */ + for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { + int const bucketSize = rankPosition[n].curr - rankPosition[n].base; + U32 const bucketStartIdx = rankPosition[n].base; + if (bucketSize > 1) { + assert(bucketStartIdx < maxSymbolValue1); + HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); + } + } + + assert(HUF_isSorted(huffNode, maxSymbolValue1)); +} + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) + +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. + * + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). + */ +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) +{ + nodeElt* const huffNode0 = huffNode - 1; + int nonNullRank; + int lowS, lowN; + int nodeNb = STARTNODE; + int n, nodeRoot; + DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); + /* init for parents */ + nonNullRank = (int)maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); + + return nonNullRank; +} + +/** + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. + */ +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) +{ + HUF_CElt* const ct = CTable + 1; + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; nhuffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); + + DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); + + /* safety checks */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + + /* determine and enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + + HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + + return maxNbBits; +} + +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + HUF_CElt const* ct = CTable + 1; + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += HUF_getNbBits(ct[s]) * count[s]; + } + return nbBits >> 3; +} + +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + HUF_CTableHeader header = HUF_readCTableHeader(CTable); + HUF_CElt const* ct = CTable + 1; + int bad = 0; + int s; + + assert(header.tableLog <= HUF_TABLELOG_ABSOLUTEMAX); + + if (header.maxSymbolValue < maxSymbolValue) + return 0; + + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); + } + return !bad; +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +/** HUF_CStream_t: + * Huffman uses its own BIT_CStream_t implementation. + * There are three major differences from BIT_CStream_t: + * 1. HUF_addBits() takes a HUF_CElt (size_t) which is + * the pair (nbBits, value) in the format: + * format: + * - Bits [0, 4) = nbBits + * - Bits [4, 64 - nbBits) = 0 + * - Bits [64 - nbBits, 64) = value + * 2. The bitContainer is built from the upper bits and + * right shifted. E.g. to add a new value of N bits + * you right shift the bitContainer by N, then or in + * the new value into the N upper bits. + * 3. The bitstream has two bit containers. You can add + * bits to the second container and merge them into + * the first container. + */ + +#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) + +typedef struct { + size_t bitContainer[2]; + size_t bitPos[2]; + + BYTE* startPtr; + BYTE* ptr; + BYTE* endPtr; +} HUF_CStream_t; + +/**! HUF_initCStream(): + * Initializes the bitstream. + * @returns 0 or an error code. + */ +static size_t HUF_initCStream(HUF_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + ZSTD_memset(bitC, 0, sizeof(*bitC)); + bitC->startPtr = (BYTE*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); + if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! HUF_addBits(): + * Adds the symbol stored in HUF_CElt elt to the bitstream. + * + * @param elt The element we're adding. This is a (nbBits, value) pair. + * See the HUF_CStream_t docs for the format. + * @param idx Insert into the bitstream at this idx. + * @param kFast This is a template parameter. If the bitstream is guaranteed + * to have at least 4 unused bits after this call it may be 1, + * otherwise it must be 0. HUF_addBits() is faster when fast is set. + */ +FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast) +{ + assert(idx <= 1); + assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX); + /* This is efficient on x86-64 with BMI2 because shrx + * only reads the low 6 bits of the register. The compiler + * knows this and elides the mask. When fast is set, + * every operation can use the same value loaded from elt. + */ + bitC->bitContainer[idx] >>= HUF_getNbBits(elt); + bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); + /* We only read the low 8 bits of bitC->bitPos[idx] so it + * doesn't matter that the high bits have noise from the value. + */ + bitC->bitPos[idx] += HUF_getNbBitsFast(elt); + assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + /* The last 4-bits of elt are dirty if fast is set, + * so we must not be overwriting bits that have already been + * inserted into the bit container. + */ +#if DEBUGLEVEL >= 1 + { + size_t const nbBits = HUF_getNbBits(elt); + size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; + (void)dirtyBits; + /* Middle bits are 0. */ + assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); + /* We didn't overwrite any bits in the bit container. */ + assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; + } +#endif +} + +FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) +{ + bitC->bitContainer[1] = 0; + bitC->bitPos[1] = 0; +} + +/*! HUF_mergeIndex1() : + * Merges the bit container @ index 1 into the bit container @ index 0 + * and zeros the bit container @ index 1. + */ +FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) +{ + assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); + bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); + bitC->bitContainer[0] |= bitC->bitContainer[1]; + bitC->bitPos[0] += bitC->bitPos[1]; + assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); +} + +/*! HUF_flushBits() : +* Flushes the bits in the bit container @ index 0. +* +* @post bitPos will be < 8. +* @param kFast If kFast is set then we must know a-priori that +* the bit container will not overflow. +*/ +FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) +{ + /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ + size_t const nbBits = bitC->bitPos[0] & 0xFF; + size_t const nbBytes = nbBits >> 3; + /* The top nbBits bits of bitContainer are the ones we need. */ + size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); + /* Mask bitPos to account for the bytes we consumed. */ + bitC->bitPos[0] &= 7; + assert(nbBits > 0); + assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitContainer); + bitC->ptr += nbBytes; + assert(!kFast || bitC->ptr <= bitC->endPtr); + if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + /* bitContainer doesn't need to be modified because the leftover + * bits are already the top bitPos bits. And we don't care about + * noise in the lower values. + */ +} + +/*! HUF_endMark() + * @returns The Huffman stream end mark: A 1-bit value = 1. + */ +static HUF_CElt HUF_endMark(void) +{ + HUF_CElt endMark; + HUF_setNbBits(&endMark, 1); + HUF_setValue(&endMark, 1); + return endMark; +} + +/*! HUF_closeCStream() : + * @return Size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +static size_t HUF_closeCStream(HUF_CStream_t* bitC) +{ + HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); + HUF_flushBits(bitC, /* kFast */ 0); + { + size_t const nbBits = bitC->bitPos[0] & 0xFF; + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); + } +} + +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) +{ + HUF_addBits(bitCPtr, CTable[symbol], idx, fast); +} + +FORCE_INLINE_TEMPLATE void +HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, + const BYTE* ip, size_t srcSize, + const HUF_CElt* ct, + int kUnroll, int kFastFlush, int kLastFast) +{ + /* Join to kUnroll */ + int n = (int)srcSize; + int rem = n % kUnroll; + if (rem > 0) { + for (; rem > 0; --rem) { + HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); + } + HUF_flushBits(bitC, kFastFlush); + } + assert(n % kUnroll == 0); + + /* Join to 2 * kUnroll */ + if (n % (2 * kUnroll)) { + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); + HUF_flushBits(bitC, kFastFlush); + n -= kUnroll; + } + assert(n % (2 * kUnroll) == 0); + + for (; n>0; n-= 2 * kUnroll) { + /* Encode kUnroll symbols into the bitstream @ index 0. */ + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); + HUF_flushBits(bitC, kFastFlush); + /* Encode kUnroll symbols into the bitstream @ index 1. + * This allows us to start filling the bit container + * without any data dependencies. + */ + HUF_zeroIndex1(bitC); + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); + /* Merge bitstream @ index 1 into the bitstream @ index 0 */ + HUF_mergeIndex1(bitC); + HUF_flushBits(bitC, kFastFlush); + } + assert(n == 0); + +} + +/** + * Returns a tight upper bound on the output space needed by Huffman + * with 8 bytes buffer to handle over-writes. If the output is at least + * this large we don't need to do bounds checks during Huffman encoding. + */ +static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) +{ + return ((srcSize * tableLog) >> 3) + 8; +} + + +FORCE_INLINE_TEMPLATE size_t +HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + U32 const tableLog = HUF_readCTableHeader(CTable).tableLog; + HUF_CElt const* ct = CTable + 1; + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + HUF_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { BYTE* op = ostart; + size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); + if (HUF_isError(initErr)) return 0; } + + if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); + else { + if (MEM_32bits()) { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: ZSTD_FALLTHROUGH; + case 9: ZSTD_FALLTHROUGH; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 7: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } else { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 9: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 7: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 6: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } + } + assert(bitC.ptr <= bitC.endPtr); + + return HUF_closeCStream(&bitC); +} + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + if (flags & HUF_flags_bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); + } + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); +} + +#else + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + (void)flags; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +#endif + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +static size_t +HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, int flags) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + op += cSize; + } + + return (size_t)(op-ostart); +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) +{ + size_t const cSize = (nbStreams==HUF_singleStream) ? + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + assert(op >= ostart); + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return (size_t)(op-ostart); +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + } wksps; +} HUF_compress_tables_t; + +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ + +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + int flags) +{ + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); + + if (!(flags & HUF_flags_optimalDepth)) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + const unsigned minTableLog = HUF_minTableLog(symbolCardinality); + size_t optSize = ((size_t) ~0) - 1; + unsigned optLog = maxTableLog, optLogGuess; + + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + + /* Search until size increases */ + for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); + + { size_t maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + if (maxBits < optLogGuess && optLogGuess > minTableLog) break; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + } + + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize > optSize + 1) { + break; + } + + if (newSize < optSize) { + optSize = newSize; + optLog = optLogGuess; + } + } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; + } +} + +/* HUF_compress_internal() : + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ +static size_t +HUF_compress_internal (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + HUF_nbStreams_e nbStreams, + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); + HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); + + /* checks & inits */ + if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); + if (!srcSize) return 0; /* Uncompressed */ + if (!dstSize) return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* If uncompressible data is suspected, do a smaller sampling first */ + DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); + if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + size_t largestTotal = 0; + DEBUGLOG(5, "input suspected incompressible : sampling to check"); + { unsigned maxSymbolValueBegin = maxSymbolValue; + CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestBegin; + } + { unsigned maxSymbolValueEnd = maxSymbolValue; + CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestEnd; + } + if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); + + /* Check validity of previous table */ + if ( repeat + && *repeat == HUF_repeat_check + && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, table->CTable, flags); +} + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, hufTable, + repeat, flags); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * consider skipping quickly + * reuse an existing huffman compression table */ +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + hufTable, repeat, flags); +} diff --git a/tools/zstd/lib/compress/zstd_compress.c b/tools/zstd/lib/compress/zstd_compress.c new file mode 100644 index 00000000..9284e2a4 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress.c @@ -0,0 +1,7153 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ +#include "../common/mem.h" +#include "hist.h" /* HIST_countFast_wksp */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" +#include "zstd_compress_superblock.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */ + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ +#ifndef ZSTD_COMPRESS_HEAPMODE +# define ZSTD_COMPRESS_HEAPMODE 0 +#endif + +/*! + * ZSTD_HASHLOG3_MAX : + * Maximum size of the hash table dedicated to find 3-bytes matches, + * in log format, aka 17 => 1 << 17 == 128Ki positions. + * This structure is only used in zstd_opt. + * Since allocation is centralized for all strategies, it has to be known here. + * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3, + * so that zstd_opt.c doesn't need to know about this constant. + */ +#ifndef ZSTD_HASHLOG3_MAX +# define ZSTD_HASHLOG3_MAX 17 +#endif + +/*-************************************* +* Helper functions +***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only valid for + * the one-pass compression functions. + * When employing the streaming mode, + * if flushes are frequently altering the size of blocks, + * the overhead from block headers can make the compressed data larger + * than the return value of ZSTD_compressBound(). + */ +size_t ZSTD_compressBound(size_t srcSize) { + size_t const r = ZSTD_COMPRESSBOUND(srcSize); + if (r==0) return ERROR(srcSize_wrong); + return r; +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + const void* dictContent; + size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; + ZSTD_matchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + ZSTD_memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) return NULL; + + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); + cctx->staticSize = workspaceSize; + + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +/** + * Clears and frees all of the dictionaries in the CCtx. + */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} + +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} + +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_clearAllDicts(cctx); +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; +#endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx); + if (cctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); + { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); + } + return 0; +} + + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void)cctx; + return 0; +#endif +} + + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + + ZSTD_sizeof_localDict(cctx->localDict) + + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +/* Returns true if the strategy supports using a row based matchfinder */ +static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { + return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); +} + +/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder + * for this compression. + */ +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); +} + +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { +#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON) + int const kHasSIMD128 = 1; +#else + int const kHasSIMD128 = 0; +#endif + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; + if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; + if (kHasSIMD128) { + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; + } else { + if (cParams->windowLog > 17) mode = ZSTD_ps_enable; + } + return mode; +} + +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ +static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 forDDSDict) { + assert(useRowMatchFinder != ZSTD_ps_auto); + /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. + * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. + */ + return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); +} + +/* Returns ZSTD_ps_enable if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns ZSTD_ps_disable otherwise. + */ +static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +static int ZSTD_resolveExternalSequenceValidation(int mode) { + return mode; +} + +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} + +static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) { + if (value != ZSTD_ps_auto) return value; + if (cLevel < 10) { + return ZSTD_ps_disable; + } else { + return ZSTD_ps_enable; + } +} + +/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. + * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ +static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); + cctxParams.cParams = cParams; + + /* Adjust advanced params according to cParams */ + cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); + if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); + cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, + cctxParams.compressionLevel); + assert(!ZSTD_checkCParams(cParams)); + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); + params->customMem = customMem; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_customFree(params, params->customMem); + return 0; +} + +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; +} + +#define ZSTD_NO_CLEVEL 0 + +/** + * Initializes `cctxParams` from `params` and `compressionLevel`. + * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. + */ +static void +ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, + const ZSTD_parameters* params, + int compressionLevel) +{ + assert(!ZSTD_checkCParams(params->cParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = compressionLevel; + cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); + cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); + cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); + cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", + cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); +} + +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return 0; +} + +/** + * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. + * @param params Validated zstd parameters. + */ +static void ZSTD_CCtxParams_setZstdParams( + ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) +{ + assert(!ZSTD_checkCParams(params->cParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = ZSTD_NO_CLEVEL; +} + +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_overlapLog: +#ifdef ZSTD_MULTITHREAD + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; +#else + bounds.lowerBound = 0; + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; + + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_useBlockSplitter: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_useRowMatchFinder: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_prefetchCDictTables: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_enableSeqProducerFallback: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_searchForExternalRepcodes: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } +} + +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. + */ +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; +} + +#define BOUNDCHECK(cParam, val) \ + do { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ + } while (0) + + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch(param) + { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: + default: + return 0; + } +} + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); + } } + + switch(param) + { + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; + + case ZSTD_c_compressionLevel: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: + break; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); + switch(param) + { + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel : { + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else + CCtxParams->compressionLevel = value; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ + } + + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = (U32)value; + return CCtxParams->cParams.windowLog; + + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = (U32)value; + return CCtxParams->cParams.hashLog; + + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = (U32)value; + return CCtxParams->cParams.chainLog; + + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; + + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = (U32)value; + return CCtxParams->cParams.minMatch; + + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = (U32)value; + return CCtxParams->cParams.targetLength; + + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; + + case ZSTD_c_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return (size_t)CCtxParams->fParams.contentSizeFlag; + + case ZSTD_c_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return (size_t)CCtxParams->fParams.checksumFlag; + + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; + + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); + return (size_t)CCtxParams->forceWindow; + + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } + + case ZSTD_c_literalCompressionMode : { + const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } + + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + CCtxParams->nbWorkers = value; + return (size_t)(CCtxParams->nbWorkers); +#endif + + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + /* Adjust to the minimum non-default value. */ + if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + assert(value >= 0); + CCtxParams->jobSize = value; + return CCtxParams->jobSize; +#endif + + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->overlapLog = value; + return (size_t)CCtxParams->overlapLog; +#endif + + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->rsyncable = value; + return (size_t)CCtxParams->rsyncable; +#endif + + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return (size_t)CCtxParams->enableDedicatedDictSearch; + + case ZSTD_c_enableLongDistanceMatching : + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); + CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; + return CCtxParams->ldmParams.enableLdm; + + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = (U32)value; + return CCtxParams->ldmParams.hashLog; + + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = (U32)value; + return CCtxParams->ldmParams.minMatchLength; + + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = (U32)value; + return CCtxParams->ldmParams.bucketSizeLog; + + case ZSTD_c_ldmHashRateLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); + CCtxParams->ldmParams.hashRateLog = (U32)value; + return CCtxParams->ldmParams.hashRateLog; + + case ZSTD_c_targetCBlockSize : + if (value!=0) { /* 0 ==> default */ + value = MAX(value, ZSTD_TARGETCBLOCKSIZE_MIN); + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + } + CCtxParams->targetCBlockSize = (U32)value; + return CCtxParams->targetCBlockSize; + + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return (size_t)CCtxParams->srcSizeHint; + + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; + + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; + + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + return CCtxParams->blockDelimiters; + + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return (size_t)CCtxParams->validateSequences; + + case ZSTD_c_useBlockSplitter: + BOUNDCHECK(ZSTD_c_useBlockSplitter, value); + CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value; + return CCtxParams->useBlockSplitter; + + case ZSTD_c_useRowMatchFinder: + BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); + CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value; + return CCtxParams->useRowMatchFinder; + + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return (size_t)CCtxParams->deterministicRefPrefix; + + case ZSTD_c_prefetchCDictTables: + BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); + CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value; + return CCtxParams->prefetchCDictTables; + + case ZSTD_c_enableSeqProducerFallback: + BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); + CCtxParams->enableMatchFinderFallback = value; + return (size_t)CCtxParams->enableMatchFinderFallback; + + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + CCtxParams->maxBlockSize = value; + return CCtxParams->maxBlockSize; + + case ZSTD_c_searchForExternalRepcodes: + BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value); + CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value; + return CCtxParams->searchForExternalRepcodes; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } +} + +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_getParameter( + ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch(param) + { + case ZSTD_c_format : + *value = CCtxParams->format; + break; + case ZSTD_c_compressionLevel : + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog : + *value = (int)CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog : + *value = (int)CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog : + *value = (int)CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog : + *value = CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch : + *value = CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength : + *value = CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy : + *value = (unsigned)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag : + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag : + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag : + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow : + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict : + *value = CCtxParams->attachDictPref; + break; + case ZSTD_c_literalCompressionMode : + *value = CCtxParams->literalCompressionMode; + break; + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + assert(CCtxParams->nbWorkers == 0); +#endif + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; + break; +#endif + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->overlapLog; + break; +#endif + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; + case ZSTD_c_enableLongDistanceMatching : + *value = CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog : + *value = CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch : + *value = CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog : + *value = CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog : + *value = CCtxParams->ldmParams.hashRateLog; + break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; + case ZSTD_c_useBlockSplitter : + *value = (int)CCtxParams->useBlockSplitter; + break; + case ZSTD_c_useRowMatchFinder : + *value = (int)CCtxParams->useRowMatchFinder; + break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; + case ZSTD_c_prefetchCDictTables: + *value = (int)CCtxParams->prefetchCDictTables; + break; + case ZSTD_c_enableSeqProducerFallback: + *value = CCtxParams->enableMatchFinderFallback; + break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; + case ZSTD_c_searchForExternalRepcodes: + *value = (int)CCtxParams->searchForExternalRepcodes; + break; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return 0; +} + +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); + + cctx->requestedParams = *params; + return 0; +} + +size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) +{ + ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setCParams"); + /* only update if all parameters are valid */ + FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), ""); + return 0; +} + +size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) +{ + ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setFParams"); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); + return 0; +} + +size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParams"); + /* First check cParams, because we want to update all or none. */ + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + /* Next set fParams, because this could fail if the cctx isn't in init stage. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); + /* Finally set cParams, which should succeed. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); + return 0; +} + +size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); + +/** + * Initializes the local dictionary using requested parameters. + * NOTE: Initialization does not employ the pledged src size, + * because the dictionary may be used for multiple compressions. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + /* Local dictionary already initialized. */ + assert(cctx->cdict == dl->cdict); + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); + + dl->cdict = ZSTD_createCDict_advanced2( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + &cctx->requestedParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->cdict = dl->cdict; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when cctx is not in init stage."); + ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ + if (dict == NULL || dictSize == 0) /* no dictionary */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; + } else { + /* copy dictionary content inside CCtx to own its lifetime */ + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "static CCtx can't allocate for an internal copy of dictionary"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, + "allocation failed for dictionary content"); + ZSTD_memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ + cctx->localDict.dict = dictBuffer; /* read-only reference */ + } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); + cctx->cdict = cdict; + return 0; +} + +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); + ZSTD_clearAllDicts(cctx); + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } + return 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Reset parameters is only possible during init stage."); + ZSTD_clearAllDicts(cctx); + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} + + +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP_TYPE(cParam, val, type) \ + do { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } while (0) +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); + return cParams; +} + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +{ + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ + { + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } + } +} + +/** ZSTD_adjustCParams_internal() : + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. + * note : `srcSize==0` means 0! + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize, + ZSTD_cParamMode_e mode, + ZSTD_paramSwitch_e useRowMatchFinder) +{ + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); + + /* Cascade the selected strategy down to the next-highest one built into + * this binary. */ +#ifdef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btultra2) { + cPar.strategy = ZSTD_btultra; + } + if (cPar.strategy == ZSTD_btultra) { + cPar.strategy = ZSTD_btopt; + } +#endif +#ifdef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btopt) { + cPar.strategy = ZSTD_btlazy2; + } +#endif +#ifdef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_btlazy2) { + cPar.strategy = ZSTD_lazy2; + } +#endif +#ifdef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_lazy2) { + cPar.strategy = ZSTD_lazy; + } +#endif +#ifdef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_lazy) { + cPar.strategy = ZSTD_greedy; + } +#endif +#ifdef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_greedy) { + cPar.strategy = ZSTD_dfast; + } +#endif +#ifdef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + if (cPar.strategy == ZSTD_dfast) { + cPar.strategy = ZSTD_fast; + cPar.targetLength = 0; + } +#endif + + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + /* If we don't know the source size, don't make any + * assumptions about it. We will already have selected + * smaller parameters if a dictionary is in use. + */ + break; + case ZSTD_cpm_createCDict: + /* Assume a small source size when creating a dictionary + * with an unknown source size. + */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; + break; + case ZSTD_cpm_attachDict: + /* Dictionary has its own dedicated parameters which have + * already been selected. We are selecting parameters + * for only the source. + */ + dictSize = 0; + break; + default: + assert(0); + break; + } + + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + + /* We can't use more than 32 bits of hash in total, so that means that we require: + * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 + */ + if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { + U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + if (cPar.hashLog > maxShortCacheHashLog) { + cPar.hashLog = maxShortCacheHashLog; + } + if (cPar.chainLog > maxShortCacheHashLog) { + cPar.chainLog = maxShortCacheHashLog; + } + } + + + /* At this point, we aren't 100% sure if we are using the row match finder. + * Unless it is explicitly disabled, conservatively assume that it is enabled. + * In this case it will only be disabled for small sources, so shrinking the + * hash log a little bit shouldn't result in any ratio loss. + */ + if (useRowMatchFinder == ZSTD_ps_auto) + useRowMatchFinder = ZSTD_ps_enable; + + /* We can't hash more than 32-bits in total. So that means that we require: + * (hashLog - rowLog + 8) <= 32 + */ + if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { + /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); + U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; + U32 const maxHashLog = maxRowHashLog + rowLog; + assert(cPar.hashLog >= rowLog); + if (cPar.hashLog > maxHashLog) { + cPar.hashLog = maxHashLog; + } + } + + return cPar; +} + +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); +} + +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); + if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); + assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); +} + +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 enableDedicatedDictSearch, + const U32 forCCtx) +{ + /* chain table size should be 0 for fast or row-hash strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((1<strategy, useRowMatchFinder) + ? ZSTD_cwksp_aligned_alloc_size(hSize) + : 0; + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) + ? optPotentialSpace + : 0; + size_t const slackSpace = ZSTD_cwksp_slack_space_required(); + + /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ + ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); + assert(useRowMatchFinder != ZSTD_ps_auto); + + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; +} + +/* Helper function for calculating memory requirements. + * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ +static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { + U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; + return blockSize / divider; +} + +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const ZSTD_paramSwitch_e useRowMatchFinder, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize, + int useSequenceProducer, + size_t maxBlockSize) +{ + size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); + + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? + ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + + + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); + + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const externalSeqSpace = useSequenceProducer + ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + : 0; + + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace + + externalSeqSpace; + + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + &cParams); + + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); + + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN, ZSTD_hasExtSeqProd(params), params->maxBlockSize); + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_getFrameProgression(cctx->mtctx); + } +#endif + { ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : + cctx->inBuffPos - cctx->inToCompress; + if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; +} } + +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_toFlushNow(cctx->mtctx); + } +#endif + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ +} + +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); +} + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + ZSTD_window_clear(&ms->window); + + ms->nextToUpdate = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; +} + +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; + +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; + +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; + +/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ +static U64 ZSTD_bitmix(U64 val, U64 len) { + val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); + val *= 0x9FB21C651E98DF25ULL; + val ^= (val >> 35) + len ; + val *= 0x9FB21C651E98DF25ULL; + return val ^ (val >> 28); +} + +/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ +static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) { + ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); +} + +static size_t +ZSTD_reset_matchState(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + const ZSTD_compressionParameters* cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) +{ + /* disable chain table allocation for fast or row-based strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, + ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + assert(useRowMatchFinder != ZSTD_ps_auto); + if (forceResetIndex == ZSTDirp_reset) { + ZSTD_window_init(&ms->window); + ZSTD_cwksp_mark_tables_dirty(ws); + } + + ms->hashLog3 = hashLog3; + ms->lazySkipping = 0; + + ZSTD_invalidateMatchState(ms); + + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize; + /* We want to generate a new salt in case we reset a Cctx, but we always want to use + * 0 when we reset a Cdict */ + if(forWho == ZSTD_resetTarget_CCtx) { + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); + ZSTD_advanceHashSalt(ms); + } else { + /* When we are not salting we want to always memset the memory */ + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ms->hashSalt = 0; + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } + } + + /* opt parser space */ + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, ZSTD_OPT_SIZE * sizeof(ZSTD_optimal_t)); + } + + ms->cParams = *cParams; + + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + return 0; +} + +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + +/** ZSTD_dictTooBig(): + * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in + * one go generically. So we ensure that in that case we reset the tables to zero, + * so that we can load as much of the dictionary as possible. + */ +static int ZSTD_dictTooBig(size_t const loadedDictSize) +{ + return loadedDictSize > ZSTD_CHUNKSIZE_MAX; +} + +/*! ZSTD_resetCCtx_internal() : + * @param loadedDictSize The size of the dictionary to be loaded + * into the context, if any. If no dictionary is used, or the + * dictionary is being attached / copied, then pass 0. + * note : `params` are assumed fully validated at this stage. + */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params const* params, + U64 const pledgedSrcSize, + size_t const loadedDictSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) +{ + ZSTD_cwksp* const ws = &zc->workspace; + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + + zc->isFirstBlock = 1; + + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->useBlockSplitter != ZSTD_ps_auto); + assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); + } + + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(params->maxBlockSize, windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, ZSTD_hasExtSeqProd(params)); + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); + + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); + ZSTD_indexResetPolicy_e needsIndexReset = + (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; + + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, + buffInSize, buffOutSize, pledgedSrcSize, ZSTD_hasExtSeqProd(params), params->maxBlockSize); + + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); + + { /* Check if workspace is large enough, alloc a new one if needed */ + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); + int resizeWorkspace = workspaceTooSmall || workspaceWasteful; + DEBUGLOG(4, "Need %zu B workspace", neededSpace); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (resizeWorkspace) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, + neededSpace >> 10); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); + + DEBUGLOG(5, "reserving object space"); + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + } } + + ZSTD_cwksp_clear(ws); + + /* init params */ + zc->blockState.matchState.cParams = params->cParams; + zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->dictContentSize = 0; + + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + /* reserve space for block-level external sequences */ + if (ZSTD_hasExtSeqProd(params)) { + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + zc->extSeqBufCapacity = maxNbExternalSeq; + zc->extSeqBuf = + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + } + + /* buffers */ + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + zc->bufferedPolicy = zbuff; + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); + + /* ldm bucketOffsets table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const numBuckets = + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); + + zc->initialized = 1; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iblockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); +} + +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize) +{ + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ +} + +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_makeClean, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); + } + + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + /* loadedDictEnd is expressed within the referential of the active context */ + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, + ZSTD_compressionParameters const* cParams) { + if (ZSTD_CDictIndicesAreTagged(cParams)){ + /* Remove tags from the CDict table if they are present. + * See docs on "short cache" in zstd_compress_internal.h for context. */ + size_t i; + for (i = 0; i < tableSize; i++) { + U32 const taggedIndex = src[i]; + U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; + dst[i] = index; + } + } else { + ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); + } +} + +static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + + assert(!cdict->matchState.dedicatedDictSearch); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + + { unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + } + + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + assert(params.useRowMatchFinder != ZSTD_ps_auto); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) + ? ((size_t)1 << cdict_cParams->chainLog) + : 0; + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize, cdict_cParams); + + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ + if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize, cdict_cParams); + } + /* copy tag table */ + if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { + size_t const tagTableSize = hSize; + ZSTD_memcpy(cctx->blockState.matchState.tagTable, + cdict->matchState.tagTable, + tagTableSize); + cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; + } + } + + /* Zero the hashTable3, since the cdict never fills it */ + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + assert(cdict->matchState.hashLog3 == 0); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + + /* copy dictionary offsets */ + { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; + ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); + + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } +} + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); + params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; + params.ldmParams = srcCCtx->appliedParams.ldmParams; + params.fParams = fParams; + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + } + + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, + srcCCtx->appliedParams.useRowMatchFinder, + 0 /* forDDSDict */) + ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) + : 0; + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + + /* copy dictionary offsets */ + { + const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + dstCCtx->dictContentSize = srcCCtx->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + fParams, pledgedSrcSize, + zbuff); +} + + +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void +ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ + U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; + assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U<<31)); /* can be casted to int */ + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table reuse logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { + int column; + for (column=0; columncParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } + + if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } + + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + int longOffsets = 0; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; + } + if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; +} + +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); +} + +/* ZSTD_blockSplitterEnabled(): + * Returns if block splitting param is being used + * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter); + assert(cctxParams->useBlockSplitter != ZSTD_ps_auto); + return (cctxParams->useBlockSplitter == ZSTD_ps_enable); +} + +/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types + * and size of the sequences statistics + */ +typedef struct { + U32 LLtype; + U32 Offtype; + U32 MLtype; + size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; +} ZSTD_symbolEncodingTypeStats_t; + +/* ZSTD_buildSequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. + * Modifies `nextEntropy` to have the appropriate values as a side effect. + * nbSeq must be greater than 0. + * + * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildSequencesStatistics( + const seqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) +{ + BYTE* const ostart = dst; + const BYTE* const oend = dstEnd; + BYTE* op = ostart; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + ZSTD_symbolEncodingTypeStats_t stats; + + stats.lastCountSize = 0; + /* convert length/distances into codes */ + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); + assert(op <= oend); + assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype, + countWorkspace, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, + sizeof(prevEntropy->litlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); + stats.size = countSize; + return stats; + } + if (stats.LLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype, + countWorkspace, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, + sizeof(prevEntropy->offcodeCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); + stats.size = countSize; + return stats; + } + if (stats.Offtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype, + countWorkspace, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, + sizeof(prevEntropy->matchlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); + stats.size = countSize; + return stats; + } + if (stats.MLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + stats.size = (size_t)(op-ostart); + return stats; +} + +/* ZSTD_entropyCompressSeqStore_internal(): + * compresses both literals and sequences + * Returns compressed size of block, or a zstd error. + */ +#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore_internal( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned* count = (unsigned*)entropyWorkspace; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + const seqDef* const sequences = seqStorePtr->sequencesStart; + const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t lastCountSize; + int longOffsets = 0; + + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); + + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); + + /* Compress literals */ + { const BYTE* const literals = seqStorePtr->litStart; + size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + size_t const litSize = (size_t)(seqStorePtr->lit - literals); + + size_t const cSize = ZSTD_compressLiterals( + op, dstCapacity, + literals, litSize, + entropyWorkspace, entropyWkspSize, + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); + assert(cSize <= dstCapacity); + op += cSize; + } + + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); + if (nbSeq < 128) { + *op++ = (BYTE)nbSeq; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } + assert(op <= oend); + if (nbSeq==0) { + /* Copy the old tables over as if we repeated them */ + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return (size_t)(op - ostart); + } + { BYTE* const seqHead = op++; + /* build stats for sequences */ + const ZSTD_symbolEncodingTypeStats_t stats = + ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; + op += stats.size; + longOffsets = stats.longOffsets; + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + assert(op <= oend); + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } + + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return (size_t)(op - ostart); +} + +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + size_t const cSize = ZSTD_entropyCompressSeqStore_internal( + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + dst, dstCapacity, + entropyWorkspace, entropyWkspSize, bmi2); + if (cSize == 0) return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) { + DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); + return 0; /* block not compressed */ + } + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); + + /* Check compressibility */ + { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) return 0; /* block not compressed */ + } + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. + * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. + */ + assert(cSize < ZSTD_BLOCKSIZE_MAX); + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) +{ + static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_COMPRESSBLOCK_DOUBLEFAST, + ZSTD_COMPRESSBLOCK_GREEDY, + ZSTD_COMPRESSBLOCK_LAZY, + ZSTD_COMPRESSBLOCK_LAZY2, + ZSTD_COMPRESSBLOCK_BTLAZY2, + ZSTD_COMPRESSBLOCK_BTOPT, + ZSTD_COMPRESSBLOCK_BTULTRA, + ZSTD_COMPRESSBLOCK_BTULTRA2 + }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT, + ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT, + ZSTD_COMPRESSBLOCK_LAZY_EXTDICT, + ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT, + ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT, + ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT, + ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT, + ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT + }, + { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE, + ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE + }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH, + ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH, + ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH, + NULL, + NULL, + NULL, + NULL } + }; + ZSTD_blockCompressor selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); + if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { + static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { + { + ZSTD_COMPRESSBLOCK_GREEDY_ROW, + ZSTD_COMPRESSBLOCK_LAZY_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW, + ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW, + ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW + }, + { + ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW, + ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW, + ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW + } + }; + DEBUGLOG(4, "Selecting a row-based matchfinder"); + assert(useRowMatchFinder != ZSTD_ps_auto); + selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; + } else { + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + } + assert(selectedCompressor != NULL); + return selectedCompressor; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthType = ZSTD_llt_none; +} + +/* ZSTD_postProcessSequenceProducerResult() : + * Validates and post-processes sequences obtained through the external matchfinder API: + * - Checks whether nbExternalSeqs represents an error condition. + * - Appends a block delimiter to outSeqs if one is not already present. + * See zstd.h for context regarding block delimiters. + * Returns the number of sequences after post-processing, or an error code. */ +static size_t ZSTD_postProcessSequenceProducerResult( + ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize +) { + RETURN_ERROR_IF( + nbExternalSeqs > outSeqsCapacity, + sequenceProducer_failed, + "External sequence producer returned error code %lu", + (unsigned long)nbExternalSeqs + ); + + RETURN_ERROR_IF( + nbExternalSeqs == 0 && srcSize > 0, + sequenceProducer_failed, + "Got zero sequences from external sequence producer for a non-empty src buffer!" + ); + + if (srcSize == 0) { + ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); + return 1; + } + + { + ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; + + /* We can return early if lastSeq is already a block delimiter. */ + if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { + return nbExternalSeqs; + } + + /* This error condition is only possible if the external matchfinder + * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ + RETURN_ERROR_IF( + nbExternalSeqs == outSeqsCapacity, + sequenceProducer_failed, + "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" + ); + + /* lastSeq is not a block delimiter, so we need to append one. */ + ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); + return nbExternalSeqs + 1; + } +} + +/* ZSTD_fastSequenceLengthSum() : + * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. + * Similar to another function in zstd_compress.c (determine_blockSize), + * except it doesn't check for a block delimiter to end summation. + * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). + * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ +static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { + size_t matchLenSum, litLenSum, i; + matchLenSum = 0; + litLenSum = 0; + for (i = 0; i < seqBufSize; i++) { + litLenSum += seqBuf[i].litLength; + matchLenSum += seqBuf[i].matchLength; + } + return litLenSum + matchLenSum; +} + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + + /* limited update after a very long match */ + { const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 curr = (U32)(istart-base); + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + } + + /* select and store sequences */ + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(&zc->appliedParams), + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + &zc->appliedParams.ldmParams, + src, srcSize), ""); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else if (ZSTD_hasExtSeqProd(&zc->appliedParams)) { + assert( + zc->extSeqBufCapacity >= ZSTD_sequenceBound(srcSize) + ); + assert(zc->appliedParams.extSeqProdFunc != NULL); + + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + + size_t const nbExternalSeqs = (zc->appliedParams.extSeqProdFunc)( + zc->appliedParams.extSeqProdState, + zc->extSeqBuf, + zc->extSeqBufCapacity, + src, srcSize, + NULL, 0, /* dict and dictSize, currently not supported */ + zc->appliedParams.compressionLevel, + windowSize + ); + + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->extSeqBuf, + nbExternalSeqs, + zc->extSeqBufCapacity, + srcSize + ); + + /* Return early if there is no error, since we don't need to worry about last literals */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_sequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->extSeqBuf, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); + FORWARD_IF_ERROR( + ZSTD_copySequencesToSeqStoreExplicitBlockDelim( + zc, &seqPos, + zc->extSeqBuf, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes + ), + "Failed to copy external sequences to seqStore!" + ); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); + return ZSTDbss_compress; + } + + /* Propagate the error if fallback is disabled */ + if (!zc->appliedParams.enableMatchFinderFallback) { + return nbPostProcessedSeqs; + } + + /* Fallback to software matchfinder */ + { ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG( + 5, + "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs + ); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } } + } else { /* not long range mode and no external matchfinder */ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor( + zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } } + return ZSTDbss_compress; +} + +static size_t ZSTD_copyBlockSequences(SeqCollector* seqCollector, const seqStore_t* seqStore, const U32 prevRepcodes[ZSTD_REP_NUM]) +{ + const seqDef* inSeqs = seqStore->sequencesStart; + const size_t nbInSequences = seqStore->sequences - inSeqs; + const size_t nbInLiterals = (size_t)(seqStore->lit - seqStore->litStart); + + ZSTD_Sequence* outSeqs = seqCollector->seqIndex == 0 ? seqCollector->seqStart : seqCollector->seqStart + seqCollector->seqIndex; + const size_t nbOutSequences = nbInSequences + 1; + size_t nbOutLiterals = 0; + repcodes_t repcodes; + size_t i; + + /* Bounds check that we have enough space for every input sequence + * and the block delimiter + */ + assert(seqCollector->seqIndex <= seqCollector->maxSequences); + RETURN_ERROR_IF( + nbOutSequences > (size_t)(seqCollector->maxSequences - seqCollector->seqIndex), + dstSize_tooSmall, + "Not enough space to copy sequences"); + + ZSTD_memcpy(&repcodes, prevRepcodes, sizeof(repcodes)); + for (i = 0; i < nbInSequences; ++i) { + U32 rawOffset; + outSeqs[i].litLength = inSeqs[i].litLength; + outSeqs[i].matchLength = inSeqs[i].mlBase + MINMATCH; + outSeqs[i].rep = 0; + + /* Handle the possible single length >= 64K + * There can only be one because we add MINMATCH to every match length, + * and blocks are at most 128K. + */ + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { + outSeqs[i].matchLength += 0x10000; + } + } + + /* Determine the raw offset given the offBase, which may be a repcode. */ + if (OFFBASE_IS_REPCODE(inSeqs[i].offBase)) { + const U32 repcode = OFFBASE_TO_REPCODE(inSeqs[i].offBase); + assert(repcode > 0); + outSeqs[i].rep = repcode; + if (outSeqs[i].litLength != 0) { + rawOffset = repcodes.rep[repcode - 1]; + } else { + if (repcode == 3) { + assert(repcodes.rep[0] > 1); + rawOffset = repcodes.rep[0] - 1; + } else { + rawOffset = repcodes.rep[repcode]; + } + } + } else { + rawOffset = OFFBASE_TO_OFFSET(inSeqs[i].offBase); + } + outSeqs[i].offset = rawOffset; + + /* Update repcode history for the sequence */ + ZSTD_updateRep(repcodes.rep, + inSeqs[i].offBase, + inSeqs[i].litLength == 0); + + nbOutLiterals += outSeqs[i].litLength; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(nbInLiterals >= nbOutLiterals); + { + const size_t lastLLSize = nbInLiterals - nbOutLiterals; + outSeqs[nbInSequences].litLength = (U32)lastLLSize; + outSeqs[nbInSequences].matchLength = 0; + outSeqs[nbInSequences].offset = 0; + assert(nbOutSequences == nbInSequences + 1); + } + seqCollector->seqIndex += nbOutSequences; + assert(seqCollector->seqIndex <= seqCollector->maxSequences); + + return 0; +} + +size_t ZSTD_sequenceBound(size_t srcSize) { + const size_t maxNbSeq = (srcSize / ZSTD_MINMATCH_MIN) + 1; + const size_t maxNbDelims = (srcSize / ZSTD_BLOCKSIZE_MAX_MIN) + 1; + return maxNbSeq + maxNbDelims; +} + +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + { + int targetCBlockSize; + FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetCBlockSize, &targetCBlockSize), ""); + RETURN_ERROR_IF(targetCBlockSize != 0, parameter_unsupported, "targetCBlockSize != 0"); + } + { + int nbWorkers; + FORWARD_IF_ERROR(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers), ""); + RETURN_ERROR_IF(nbWorkers != 0, parameter_unsupported, "nbWorkers != 0"); + } + + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + { + const size_t ret = ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + FORWARD_IF_ERROR(ret, "ZSTD_compress2 failed"); + } + assert(zc->seqCollector.seqIndex <= ZSTD_sequenceBound(srcSize)); + return zc->seqCollector.seqIndex; +} + +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } + } + return out; +} + +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; + size_t i; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + size_t u; + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } } } + return 1; +} + +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(seqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + + return nbSeqs < 4 && nbLits < 10; +} + +static void +ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) +{ + ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; + bs->prevCBlock = bs->nextCBlock; + bs->nextCBlock = tmp; +} + +/* Writes the block header */ +static void +writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) +{ + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); +} + +/** ZSTD_buildBlockEntropyStats_literals() : + * Builds entropy for the literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * Requires ENTROPY_WORKSPACE_SIZE workspace + * @return : size of huffman description table, or an error code + */ +static size_t +ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize, + int hufFlags) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + unsigned huffLog = LitHufLog; + HUF_repeat repeat = prevHuf->repeatMode; + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (literalsCompressionIsDisabled) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +#ifndef COMPRESS_LITERALS_SIZE_MIN +# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ +#endif + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Scan input and build symbol stats */ + { size_t const largest = + HIST_count_wksp (countWksp, &maxSymbolValue, + (const BYTE*)src, srcSize, + workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + /* only one literal symbol */ + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + /* heuristic: likely not compressible */ + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check + && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); + assert(huffLog <= LitHufLog); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + } + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } +} + + +/* ZSTD_buildDummySequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, + * and updates nextEntropy to the appropriate repeatMode. + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) +{ + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; + nextEntropy->litlength_repeatMode = FSE_repeat_none; + nextEntropy->offcode_repeatMode = FSE_repeat_none; + nextEntropy->matchlength_repeatMode = FSE_repeat_none; + return stats; +} + +/** ZSTD_buildBlockEntropyStats_sequences() : + * Builds entropy for the sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * Requires ENTROPY_WORKSPACE_SIZE wksp. + * @return : size of fse tables or error code */ +static size_t +ZSTD_buildBlockEntropyStats_sequences( + const seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + unsigned* countWorkspace = (unsigned*)workspace; + unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); + size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); + ZSTD_symbolEncodingTypeStats_t stats; + + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); + stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + prevEntropy, nextEntropy, op, oend, + strategy, countWorkspace, + entropyWorkspace, entropyWorkspaceSize) + : ZSTD_buildDummySequencesStatistics(nextEntropy); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; + fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; + fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; + return stats.size; +} + + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * Requires workspace size ENTROPY_WORKSPACE_SIZE + * @return : 0 on success, or an error code + * Note : also employed in superblock + */ +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); + int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; + + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_literalsCompressionIsDisabled(cctxParams), + workspace, wkspSize, hufFlags); + + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); + return 0; +} + +/* Returns the size estimate for the literals section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); + U32 singleStream = litSize < 256; + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ +static size_t +ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + (void)defaultMax; + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { + return nbSeq * 10; + } + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits >> 3; +} + +/* Returns the size estimate for the sequences section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +/* Returns the size estimate for a given stream of literals, of, ll, ml */ +static size_t +ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return seqSize + literalsSize + ZSTD_blockHeaderSize; +} + +/* Builds entropy statistics and uses them for blocksize estimation. + * + * @return: estimated compressed size of the seqStore, or a zstd error. + */ +static size_t +ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) +{ + ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; + DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), ""); + return ZSTD_estimateBlockSize( + seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + seqStore->ofCode, seqStore->llCode, seqStore->mlCode, + (size_t)(seqStore->sequences - seqStore->sequencesStart), + &zc->blockState.nextCBlock->entropy, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, + (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); +} + +/* Returns literals bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) +{ + size_t literalsBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef const seq = seqStore->sequencesStart[i]; + literalsBytes += seq.litLength; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { + literalsBytes += 0x10000; + } } + return literalsBytes; +} + +/* Returns match bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) +{ + size_t matchBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef seq = seqStore->sequencesStart[i]; + matchBytes += seq.mlBase + MINMATCH; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { + matchBytes += 0x10000; + } } + return matchBytes; +} + +/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). + * Stores the result in resultSeqStore. + */ +static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, + const seqStore_t* originalSeqStore, + size_t startIdx, size_t endIdx) +{ + *resultSeqStore = *originalSeqStore; + if (startIdx > 0) { + resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + } + + /* Move longLengthPos into the correct position if necessary */ + if (originalSeqStore->longLengthType != ZSTD_llt_none) { + if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { + resultSeqStore->longLengthType = ZSTD_llt_none; + } else { + resultSeqStore->longLengthPos -= (U32)startIdx; + } + } + resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; + if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { + /* This accounts for possible last literals if the derived chunk reaches the end of the block */ + assert(resultSeqStore->lit == originalSeqStore->lit); + } else { + size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; + } + resultSeqStore->llCode += startIdx; + resultSeqStore->mlCode += startIdx; + resultSeqStore->ofCode += startIdx; +} + +/** + * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. + * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). + */ +static U32 +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) +{ + U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ + assert(OFFBASE_IS_REPCODE(offBase)); + if (adjustedRepCode == ZSTD_REP_NUM) { + assert(ll0); + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 + * This is only valid if it results in a valid offset value, aka > 0. + * Note : it may happen that `rep[0]==1` in exceptional circumstances. + * In which case this function will return 0, which is an invalid offset. + * It's not an issue though, since this value will be + * compared and discarded within ZSTD_seqStore_resolveOffCodes(). + */ + return rep[0] - 1; + } + return rep[adjustedRepCode]; +} + +/** + * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise + * due to emission of RLE/raw blocks that disturb the offset history, + * and replaces any repcodes within the seqStore that may be invalid. + * + * dRepcodes are updated as would be on the decompression side. + * cRepcodes are updated exactly in accordance with the seqStore. + * + * Note : this function assumes seq->offBase respects the following numbering scheme : + * 0 : invalid + * 1-3 : repcode 1-3 + * 4+ : real_offset+3 + */ +static void +ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, + const seqStore_t* const seqStore, U32 const nbSeq) +{ + U32 idx = 0; + U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; + for (; idx < nbSeq; ++idx) { + seqDef* const seq = seqStore->sequencesStart + idx; + U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); + U32 const offBase = seq->offBase; + assert(offBase > 0); + if (OFFBASE_IS_REPCODE(offBase)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); + /* Adjust simulated decompression repcode history if we come across a mismatch. Replace + * the repcode with the offset it actually references, determined by the compression + * repcode history. + */ + if (dRawOffset != cRawOffset) { + seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); + } + } + /* Compression repcode history is always updated with values directly from the unmodified seqStore. + * Decompression repcode history may use modified seq->offset value taken from compression repcode history. + */ + ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); + ZSTD_updateRep(cRepcodes->rep, offBase, ll0); + } +} + +/* ZSTD_compressSeqStore_singleBlock(): + * Compresses a seqStore into a block with a block header, into the buffer dst. + * + * Returns the total size of that block (including header) or a ZSTD error code. + */ +static size_t +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, + const seqStore_t* const seqStore, + repcodes_t* const dRep, repcodes_t* const cRep, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock, U32 isPartition) +{ + const U32 rleMaxLength = 25; + BYTE* op = (BYTE*)dst; + const BYTE* ip = (const BYTE*)src; + size_t cSize; + size_t cSeqsSize; + + /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ + repcodes_t const dRepOriginal = *dRep; + DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); + if (isPartition) + ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); + cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); + + if (!zc->isFirstBlock && + cSeqsSize < rleMaxLength && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + cSeqsSize = 1; + } + + /* Sequence collection not supported when block splitting */ + if (zc->seqCollector.collectSequences) { + FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, seqStore, dRepOriginal.rep), "copyBlockSequences failed"); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + if (cSeqsSize == 0) { + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else if (cSeqsSize == 1) { + cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); + cSize = ZSTD_blockHeaderSize + cSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize); + } + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +/* Struct to keep track of where we are in our recursive calls. */ +typedef struct { + U32* splitLocations; /* Array of split indices */ + size_t idx; /* The current index within splitLocations being worked on */ +} seqStoreSplits; + +#define MIN_SEQUENCES_BLOCK_SPLITTING 300 + +/* Helper function to perform the recursive search for block splits. + * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. + * If advantageous to split, then we recurse down the two sub-blocks. + * If not, or if an error occurred in estimation, then we do not recurse. + * + * Note: The recursion depth is capped by a heuristic minimum number of sequences, + * defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). + * In practice, recursion depth usually doesn't go beyond 4. + * + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. + * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * maximum of 128 KB, this value is actually impossible to reach. + */ +static void +ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, + ZSTD_CCtx* zc, const seqStore_t* origSeqStore) +{ + seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + size_t estimatedOriginalSize; + size_t estimatedFirstHalfSize; + size_t estimatedSecondHalfSize; + size_t midIdx = (startIdx + endIdx)/2; + + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + assert(endIdx >= startIdx); + if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); + return; + } + ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); + ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); + estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); + estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); + estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); + DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); + if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { + return; + } + if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); + ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); + splits->splitLocations[splits->idx] = (U32)midIdx; + splits->idx++; + ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); + } +} + +/* Base recursive function. + * Populates a table with intra-block partition indices that can improve compression ratio. + * + * @return: number of splits made (which equals the size of the partition table - 1). + */ +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) +{ + seqStoreSplits splits; + splits.splitLocations = partitions; + splits.idx = 0; + if (nbSeq <= 4) { + DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); + /* Refuse to try and split anything with less than 4 sequences */ + return 0; + } + ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); + splits.splitLocations[splits.idx] = nbSeq; + DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); + return splits.idx; +} + +/* ZSTD_compressBlock_splitBlock(): + * Attempts to split a given block into multiple blocks to improve compression ratio. + * + * Returns combined size of all blocks (which includes headers), or a ZSTD error code. + */ +static size_t +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, U32 nbSeq) +{ + size_t cSize = 0; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + size_t i = 0; + size_t srcBytesTotal = 0; + U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + + /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history + * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two + * separate repcode histories that simulate repcode history on compression and decompression side, + * and use the histories to determine whether we must replace a particular repcode with its raw offset. + * + * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed + * or RLE. This allows us to retrieve the offset value that an invalid repcode references within + * a nocompress/RLE block. + * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use + * the replacement offset value rather than the original repcode to update the repcode history. + * dRep also will be the final repcode history sent to the next block. + * + * See ZSTD_seqStore_resolveOffCodes() for more details. + */ + repcodes_t dRep; + repcodes_t cRep; + ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t)); + + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + if (numSplits == 0) { + size_t cSizeSingleBlock = + ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); + FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); + assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX); + assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize); + return cSizeSingleBlock; + } + + ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); + for (i = 0; i <= numSplits; ++i) { + size_t cSizeChunk; + U32 const lastPartition = (i == numSplits); + U32 lastBlockEntireSrc = 0; + + size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + srcBytesTotal += srcBytes; + if (lastPartition) { + /* This is the final partition, need to account for possible last literals */ + srcBytes += blockSize - srcBytesTotal; + lastBlockEntireSrc = lastBlock; + } else { + ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); + } + + cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, + &dRep, &cRep, + op, dstCapacity, + ip, srcBytes, + lastBlockEntireSrc, 1 /* isPartition */); + DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", + ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); + + ip += srcBytes; + op += cSizeChunk; + dstCapacity -= cSizeChunk; + cSize += cSizeChunk; + *currSeqStore = *nextSeqStore; + assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize); + } + /* cRep and dRep may have diverged during the compression. + * If so, we use the dRep repcodes for the next block. + */ + ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); + return cSize; +} + +static size_t +ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 lastBlock) +{ + U32 nbSeq; + size_t cSize; + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); + cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); + return cSize; + } + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + } + + cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); + return cSize; +} + +static size_t +ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) +{ + /* This is an estimated upper bound for the length of an rle block. + * This isn't the actual upper bound. + * Finding the real threshold needs further investigation. + */ + const U32 rleMaxLength = 25; + size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + RETURN_ERROR_IF(zc->seqCollector.collectSequences, sequenceProducer_failed, "Uncompressible block"); + cSize = 0; + goto out; + } + } + + if (zc->seqCollector.collectSequences) { + FORWARD_IF_ERROR(ZSTD_copyBlockSequences(&zc->seqCollector, ZSTD_getSeqStore(zc), zc->blockState.prevCBlock->rep), "copyBlockSequences failed"); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + /* encode sequences and literals */ + cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, dstCapacity, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + +out: + if (!ZSTD_isError(cSize) && cSize > 1) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const size_t bss, U32 lastBlock) +{ + DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); + if (bss == ZSTDbss_compress) { + if (/* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + ZSTD_maybeRLE(&zc->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) + { + return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); + } + /* Attempt superblock compression. + * + * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the + * standard ZSTD_compressBound(). This is a problem, because even if we have + * space now, taking an extra byte now could cause us to run out of space later + * and violate ZSTD_compressBound(). + * + * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. + * + * In order to respect ZSTD_compressBound() we must attempt to emit a raw + * uncompressed block in these cases: + * * cSize == 0: Return code for an uncompressed block. + * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). + * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of + * output space. + * * cSize >= blockBound(srcSize): We have expanded the block too much so + * emit an uncompressed block. + */ + { size_t const cSize = + ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + if (cSize != ERROR(dstSize_tooSmall)) { + size_t const maxCSize = + srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); + if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return cSize; + } + } + } + } /* if (bss == ZSTDbss_compress)*/ + + DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); + /* Superblock compression failed, attempt to emit a single no compress block. + * The decoder will be able to stream this block since it is uncompressed. + */ + return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); +} + +static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock) +{ + size_t cSize = 0; + const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + + cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) +{ + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const maxDist = (U32)1 << params->cParams.windowLog; + if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); + ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + +/*! ZSTD_compress_frameChunk() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); + + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, + dstSize_tooSmall, + "not enough space to store compressed block"); + if (remaining < blockSize) blockSize = remaining; + + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; + + { size_t cSize; + if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); + assert(cSize > 0); + assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); + assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); + } else { + cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize, 1 /* frame */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + } else { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + cSize += ZSTD_blockHeaderSize; + } + } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ + + + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", + (unsigned)cSize); + } } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return (size_t)(op-ostart); +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ + BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos=0; + + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, + "dst buf is too small to fit worst-case frame header size."); + DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + if (params->format == ZSTD_f_zstd1) { + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDescriptionByte; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + +/* ZSTD_writeSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; +} + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small (stage == ZSTDcs_init); + assert(nbSeq == 0 || cctx->appliedParams.ldmParams.enableLdm != ZSTD_ps_enable); + cctx->externSeqStore.seq = seq; + cctx->externSeqStore.size = nbSeq; + cctx->externSeqStore.capacity = nbSeq; + cctx->externSeqStore.pos = 0; + cctx->externSeqStore.posInSequence = 0; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", + cctx->stage, (unsigned)srcSize); + RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, + "missing init (ZSTD_compressBegin)"); + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + assert(fhSize <= dstCapacity); + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (!srcSize) return fhSize; /* do not generate an empty block if no input */ + + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; + ms->nextToUpdate = ms->window.dictLimit; + } + if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); + } + + if (!frame) { + /* overflow check and correction for block mode */ + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); + } + + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + { size_t const cSize = frame ? + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); + FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); + cctx->consumedSrcSize += srcSize; + cctx->producedCSize += (cSize + fhSize); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + RETURN_ERROR_IF( + cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + return cSize + fhSize; + } +} + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + +/* NOTE: Must just wrap ZSTD_compressContinue_public() */ +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); +} + +static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); +} + +/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + return ZSTD_getBlockSize_deprecated(cctx); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } + + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; + + /* Assert that the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); + + { /* Ensure large dictionaries can't cause index overflow */ + + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. + * Dictionaries right at the edge will immediately trigger overflow + * correction, but I don't want to insert extra constraints here. + */ + U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + + int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); + if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { + /* Some dictionary matchfinders in zstd use "short cache", + * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each + * CDict hashtable entry as a tag rather than as part of an index. + * When short cache is used, we need to truncate the dictionary + * so that its indices don't overlap with the tag. */ + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + assert(!loadLdmDict); + } + + /* If the dictionary is too large, only load the suffix of the dictionary. */ + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); + } + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); + + DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); + + if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); + } + + /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ + if (params->cParams.strategy < ZSTD_btultra) { + U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28); + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; + + if (srcSize <= HASH_READ_SIZE) return 0; + + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); + + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm, tfp); + break; + case ZSTD_dfast: +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) + assert(srcSize >= HASH_READ_SIZE); + if (ms->dedicatedDictSearch) { + assert(ms->chainTable != NULL); + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); + } else { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ZSTD_row_update(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using row-based hash table for lazy dict"); + } else { + ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using chain-based hash table for lazy dict"); + } + } +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) + assert(srcSize >= HASH_READ_SIZE); + ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check + * and only dictionaries with 100% valid symbols can be assumed valid. + */ +static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) { + return FSE_repeat_check; + } + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) { + return FSE_repeat_check; + } + } + return FSE_repeat_valid; +} + +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize) +{ + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ + const BYTE* const dictEnd = dictPtr + dictSize; + dictPtr += 8; + bs->entropy.huf.repeatMode = HUF_repeat_check; + + { unsigned maxSymbolValue = 255; + unsigned hasZeroWeights = 1; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, + dictEnd-dictPtr, &hasZeroWeights); + + /* We only set the loaded table as valid if it contains all non-zero + * weights. Otherwise, we set it to check */ + if (!hasZeroWeights && maxSymbolValue == 255) + bs->entropy.huf.repeatMode = HUF_repeat_valid; + + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + /* fill all offset symbols to avoid garbage at end of table */ + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + bs->rep[0] = MEM_readLE32(dictPtr+0); + bs->rep[1] = MEM_readLE32(dictPtr+4); + bs->rep[2] = MEM_readLE32(dictPtr+8); + dictPtr += 12; + + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ + bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); + + /* All repCodes must be <= dictContentSize and != 0 */ + { U32 u; + for (u=0; u<3; u++) { + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); + } } } + + return dictPtr - (const BYTE*)dict; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed >= 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* dict, size_t dictSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, + void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + size_t dictID; + size_t eSize; + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); + eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); + FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); + dictPtr += eSize; + + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); + } + return dictID; +} + +/** ZSTD_compress_insertDictionary() : +* @return : dictID, or an error code */ +static size_t +ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + const ZSTD_CCtx_params* params, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, + void* workspace) +{ + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + return 0; + } + + ZSTD_reset_compressedBlockState(bs); + + /* dict restricted modes */ + if (dictContentType == ZSTD_dct_rawContent) + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_auto) { + DEBUGLOG(4, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent( + ms, ls, ws, params, dict, dictSize, dtlm, tfp); + } + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); +} + +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) + +/*! ZSTD_compressBegin_internal() : + * Assumption : either @dict OR @cdict (or none) is non-NULL, never both + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { + return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } + + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + dictContentSize, + ZSTDcrp_makeClean, zbuff) , ""); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, + cdict->dictContentSize, cdict->dictContentType, dtlm, + ZSTD_tfp_forCCtx, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= UINT_MAX); + cctx->dictID = (U32)dictID; + cctx->dictContentSize = dictContentSize; + } + return 0; +} + +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); + /* compression parameters verification and optimization */ + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); + return ZSTD_compressBegin_internal(cctx, + dict, dictSize, dictContentType, dtlm, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compressBegin_advanced_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, + NULL /*cdict*/, + &cctxParams, pledgedSrcSize); +} + +static size_t +ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtx_params cctxParams; + { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); + } + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); +} + +size_t +ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + + DEBUGLOG(4, "ZSTD_writeEpilogue"); + RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + size_t fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + ZSTD_STATIC_ASSERT(ZSTD_BLOCKHEADERSIZE == 3); + RETURN_ERROR_IF(dstCapacity<3, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE24(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) +{ +#if ZSTD_TRACE + if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { + int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + trace.dictionaryID = cctx->dictID; + trace.dictionarySize = cctx->dictContentSize; + trace.uncompressedSize = cctx->consumedSrcSize; + trace.compressedSize = cctx->producedCSize + extraCSize; + trace.params = &cctx->appliedParams; + trace.cctx = cctx; + ZSTD_trace_compress_end(cctx->traceCtx, &trace); + } + cctx->traceCtx = 0; +#else + (void)cctx; + (void)extraCSize; +#endif +} + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + DEBUGLOG(4, "end of frame : controlling src size"); + RETURN_ERROR_IF( + cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + ZSTD_CCtx_trace(cctx, endResult); + return cSize + endResult; +} + +/* NOTE: Must just wrap ZSTD_compressEnd_public() */ +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced"); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compress_advanced_internal(cctx, + dst, dstCapacity, + src, srcSize, + dict, dictSize, + &cctx->simpleApiParams); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + params, srcSize, ZSTDb_not_buffered) , ""); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel) +{ + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); + assert(params.fParams.contentSizeFlag == 1); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + } + DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); +} + +size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); + assert(cctx != NULL); + return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + size_t result; +#if ZSTD_COMPRESS_HEAPMODE + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); + result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtx(cctx); +#else + ZSTD_CCtx ctxBody; + ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ +#endif + return result; +} + + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small + * in case we are using DDS with row-hash. */ + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), + /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); +} + +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_CCtx_params params) +{ + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); + assert(!ZSTD_checkCParams(params.cParams)); + cdict->matchState.cParams = params.cParams; + cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictContent = dictBuffer; + } else { + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); + cdict->dictContent = internalBuffer; + ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + cdict->dictContentType = dictContentType; + + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + ¶ms.cParams, + params.useRowMatchFinder, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict), ""); + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is < 8 bytes. + */ + { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + params.fParams.contentSizeFlag = 1; + { size_t const dictID = ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= (size_t)(U32)-1); + cdict->dictID = (U32)dictID; + } + } + + return 0; +} + +static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, + ZSTD_paramSwitch_e useRowMatchFinder, + U32 enableDedicatedDictSearch, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); + void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; + + if (!workspace) { + ZSTD_customFree(workspace, customMem); + return NULL; + } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); + cdict->customMem = customMem; + cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ + cdict->useRowMatchFinder = useRowMatchFinder; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + ZSTD_CCtxParams_init(&cctxParams, 0); + cctxParams.cParams = cParams; + cctxParams.customMem = customMem; + return ZSTD_createCDict_advanced2( + dictBuffer, dictSize, + dictLoadMethod, dictContentType, + &cctxParams, customMem); +} + +ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams = *originalCctxParams; + ZSTD_compressionParameters cParams; + ZSTD_CDict* cdict; + + DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + if (cctxParams.enableDedicatedDictSearch) { + cParams = ZSTD_dedicatedDictSearch_getCParams( + cctxParams.compressionLevel, dictSize); + ZSTD_overrideCParams(&cParams, &cctxParams.cParams); + } else { + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { + /* Fall back to non-DDSS params */ + cctxParams.enableDedicatedDictSearch = 0; + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch); + cctxParams.cParams = cParams; + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + + cdict = ZSTD_createCDict_advanced_internal(dictSize, + dictLoadMethod, cctxParams.cParams, + cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, + customMem); + + if (!cdict || ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->customMem; + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_customFree(cdict, cMem); + } + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams) +{ + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); + /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + ZSTD_CCtx_params params; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", + (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + ZSTD_CCtxParams_init(¶ms, 0); + params.cParams = cParams; + params.useRowMatchFinder = useRowMatchFinder; + cdict->useRowMatchFinder = useRowMatchFinder; + cdict->compressionLevel = ZSTD_NO_CLEVEL; + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + params) )) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) +{ + assert(cdict != NULL); + return cdict->matchState.cParams; +} + +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; + return cdict->dictID; +} + +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); + /* Initialize the cctxParams from the cdict */ + { + ZSTD_parameters params; + params.fParams = fParams; + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); + } + /* Increase window log to fit the entire dictionary and source if the + * source size is known. Limit the increase to 19, which is the + * window log for compression level 1 with the largest source size. + */ + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); + U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; + cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); + } + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, + cdict, + &cctxParams, pledgedSrcSize, + ZSTDb_not_buffered); +} + + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + +/* ZSTD_compressBegin_usingCDict() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); +} + +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + DEBUGLOG(3, "ZSTD_createCStream"); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} + +static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +{ + if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) + return ZSTD_cpm_attachDict; + else + return ZSTD_cpm_noAttachDict; +} + +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +/*! ZSTD_initCStream_internal() : + * Note : for lib/compress only. Used by zstdmt_compress.c. + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + } else { + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + } + return 0; +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + + +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be exact. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pss) +{ + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + return 0; +} + +/*====== Compression ======*/ + +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + return cctx->blockSize - cctx->stableIn_notConsumed; + } + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); + { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; + } +} + +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants + * @return : hint size for next input to complete ongoing block */ +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) +{ + const char* const istart = (assert(input != NULL), (const char*)input->src); + const char* const iend = (istart != NULL) ? istart + input->size : istart; + const char* ip = (istart != NULL) ? istart + input->pos : istart; + char* const ostart = (assert(output != NULL), (char*)output->dst); + char* const oend = (ostart != NULL) ? ostart + output->size : ostart; + char* op = (ostart != NULL) ? ostart + output->pos : ostart; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); + assert(zcs != NULL); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { + assert(input->pos >= zcs->stableIn_notConsumed); + input->pos -= zcs->stableIn_notConsumed; + if (ip) ip -= zcs->stableIn_notConsumed; + zcs->stableIn_notConsumed = 0; + } + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + } + if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + } + if (input->src == NULL) assert(input->size == 0); + assert(input->pos <= input->size); + if (output->dst == NULL) assert(output->size == 0); + assert(output->pos <= output->size); + assert((U32)flushMode <= (U32)ZSTD_e_end); + + while (someMoreWork) { + switch(zcs->streamStage) + { + case zcss_init: + RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); + + case zcss_load: + if ( (flushMode == ZSTD_e_end) + && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd_public(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + someMoreWork = 0; break; + } + /* complete loading into inBuffer in buffered mode */ + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); + zcs->inBuffPos += loaded; + if (ip) ip += loaded; + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } else { + assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + if ( (flushMode == ZSTD_e_continue) + && ( (size_t)(iend - ip) < zcs->blockSize) ) { + /* can't compress a full block : stop here */ + zcs->stableIn_notConsumed = (size_t)(iend - ip); + ip = iend; /* pretend to have consumed input */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (ip == iend) ) { + /* empty */ + someMoreWork = 0; break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); + void* cDst; + size_t cSize; + size_t oSize = oend-op; + size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); + if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + if (inputBuffered) { + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + cSize = lastBlock ? + ZSTD_compressEnd_public(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + } else { /* !inputBuffered, hence ZSTD_bm_stable */ + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); + cSize = lastBlock ? + ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); + /* Consume the input prior to error checking to mirror buffered mode. */ + if (ip) ip += iSize; + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + if (lastBlock) assert(ip == iend); + } + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + ZSTD_FALLTHROUGH; + case zcss_flush: + DEBUGLOG(5, "flush stage"); + assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); + if (flushed) + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) return 0; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers >= 1) { + assert(cctx->mtctx != NULL); + return ZSTDMT_nextInputSizeHint(cctx->mtctx); + } +#endif + return ZSTD_nextInputSizeHint(cctx); + +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); + return ZSTD_nextInputSizeHint_MTorST(zcs); +} + +/* After a compression call set the expected input/output buffer. + * This is validated at the start of the next compression call. + */ +static void +ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) +{ + DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + cctx->expectedInBuffer = *input; + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + cctx->expectedOutBufferSize = output->size - output->pos; + } +} + +/* Validate that the input/output buffers match the expectations set by + * ZSTD_setBufferExpectations. + */ +static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, + ZSTD_outBuffer const* output, + ZSTD_inBuffer const* input, + ZSTD_EndDirective endOp) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + ZSTD_inBuffer const expect = cctx->expectedInBuffer; + if (expect.src != input->src || expect.pos != input->pos) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); + } + (void)endOp; + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + size_t const outBufferSize = output->size - output->pos; + if (cctx->expectedOutBufferSize != outBufferSize) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + } + return 0; +} + +static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, + ZSTD_EndDirective endOp, + size_t inSize) +{ + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict && !cctx->localDict.cdict) { + /* Let the cdict's compression level take priority over the requested params. + * But do not take the cdict's compression level if the "cdict" is actually a localDict + * generated from ZSTD_initLocalDict(). + */ + params.compressionLevel = cctx->cdict->compressionLevel; + } + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ + + { size_t const dictSize = prefixDict.dict + ? prefixDict.dictSize + : (cctx->cdict ? cctx->cdict->dictContentSize : 0); + ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + params.cParams = ZSTD_getCParamsFromCCtxParams( + ¶ms, cctx->pledgedSrcSizePlusOne-1, + dictSize, mode); + } + + params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams); + params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); + params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); + params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); + +#ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + ZSTD_hasExtSeqProd(¶ms) && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External sequence producer isn't supported with nbWorkers >= 1" + ); + + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { + params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ + } + if (params.nbWorkers > 0) { +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + /* mt context creation */ + if (cctx->mtctx == NULL) { + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", + params.nbWorkers); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); + } + /* mt compression */ + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); + FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); + cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; + cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; + cctx->consumedSrcSize = 0; + cctx->producedCSize = 0; + cctx->streamStage = zcss_load; + cctx->appliedParams = params; + } else +#endif /* ZSTD_MULTITHREAD */ + { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, + cctx->cdict, + ¶ms, pledgedSrcSize, + ZSTDb_buffered) , ""); + assert(cctx->appliedParams.nbWorkers == 0); + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { + /* for small input: avoid automatic flush on reaching end of block, since + * it would require to add a 3-bytes null block to end frame + */ + cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + } else { + cctx->inBuffTarget = 0; + } + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + } + return 0; +} + +/* @return provides a minimum amount of data remaining to be flushed from internal buffers + */ +size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); + /* check conditions */ + RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); + RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); + RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); + assert(cctx != NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ + size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; + if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ + && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ + && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ + if (cctx->stableIn_notConsumed) { /* not the first time */ + /* check stable source guarantees */ + RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); + RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); + } + /* pretend input was consumed, to give a sense forward progress */ + input->pos = input->size; + /* save stable inBuffer, for later control, and flush/end */ + cctx->expectedInBuffer = *input; + /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ + cctx->stableIn_notConsumed += inputSize; + /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ + return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + } + /* end of transparent initialization stage */ + + FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + size_t flushMin; + if (cctx->cParamsChanged) { + ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); + cctx->cParamsChanged = 0; + } + if (cctx->stableIn_notConsumed) { + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); + /* some early data was skipped - make it available for consumption */ + assert(input->pos >= cctx->stableIn_notConsumed); + input->pos -= cctx->stableIn_notConsumed; + cctx->stableIn_notConsumed = 0; + } + for (;;) { + size_t const ipos = input->pos; + size_t const opos = output->pos; + flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + cctx->consumedSrcSize += (U64)(input->pos - ipos); + cctx->producedCSize += (U64)(output->pos - opos); + if ( ZSTD_isError(flushMin) + || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ + if (flushMin == 0) + ZSTD_CCtx_trace(cctx, 0); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + } + FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); + + if (endOp == ZSTD_e_continue) { + /* We only require some progress with ZSTD_e_continue, not maximal progress. + * We're done if we've consumed or produced any bytes, or either buffer is + * full. + */ + if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) + break; + } else { + assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); + /* We require maximal progress. We're done when the flush is complete or the + * output buffer is full. + */ + if (flushMin == 0 || output->pos == output->size) + break; + } + } + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); + /* Either we don't require maximum forward progress, we've finished the + * flush, or we are out of output space. + */ + assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); + ZSTD_setBufferExpectations(cctx, output, input); + return flushMin; + } +#endif /* ZSTD_MULTITHREAD */ + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); + DEBUGLOG(5, "completed ZSTD_compressStream2"); + ZSTD_setBufferExpectations(cctx, output, input); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} + +size_t ZSTD_compress2(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; + ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; + DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + /* Enable stable input/output buffers. */ + cctx->requestedParams.inBufferMode = ZSTD_bm_stable; + cctx->requestedParams.outBufferMode = ZSTD_bm_stable; + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + /* Reset to the original values. */ + cctx->requestedParams.inBufferMode = originalInBufferMode; + cctx->requestedParams.outBufferMode = originalOutBufferMode; + + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + RETURN_ERROR(dstSize_tooSmall, ""); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} + +/* ZSTD_validateSequence() : + * @offCode : is presumed to follow format required by ZSTD_storeSeq() + * @returns a ZSTD error code if sequence is not valid + */ +static size_t +ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch, + size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) +{ + U32 const windowSize = 1u << windowLog; + /* posInSrc represents the amount of data the decoder would decode up to this point. + * As long as the amount of data decoded is less than or equal to window size, offsets may be + * larger than the total length of output decoded in order to reference the dict, even larger than + * window size. After output surpasses windowSize, we're limited to windowSize offsets again. + */ + size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; + RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); + /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ + RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); + return 0; +} + +/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ +static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +{ + U32 offBase = OFFSET_TO_OFFBASE(rawOffset); + + if (!ll0 && rawOffset == rep[0]) { + offBase = REPCODE1_TO_OFFBASE; + } else if (rawOffset == rep[1]) { + offBase = REPCODE_TO_OFFBASE(2 - ll0); + } else if (rawOffset == rep[2]) { + offBase = REPCODE_TO_OFFBASE(3 - ll0); + } else if (ll0 && rawOffset == rep[0] - 1) { + offBase = REPCODE3_TO_OFFBASE; + } + return offBase; +} + +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_paramSwitch_e externalRepSearch) +{ + U32 idx = seqPos->idx; + U32 const startIdx = idx; + BYTE const* ip = (BYTE const*)(src); + const BYTE* const iend = ip + blockSize; + repcodes_t updatedRepcodes; + U32 dictSize; + + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize); + + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { + U32 const litLength = inSeqs[idx].litLength; + U32 const matchLength = inSeqs[idx].matchLength; + U32 offBase; + + if (externalRepSearch == ZSTD_ps_disable) { + offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); + } else { + U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), + "Sequence validation failed"); + } + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); + ip += matchLength + litLength; + } + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + assert(externalRepSearch != ZSTD_ps_auto); + assert(idx >= startIdx); + if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { + U32* const rep = updatedRepcodes.rep; + U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ + + if (lastSeqIdx >= startIdx + 2) { + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (lastSeqIdx == startIdx + 1) { + rep[2] = rep[0]; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else { + assert(lastSeqIdx == startIdx); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[lastSeqIdx].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + if (inSeqs[idx].litLength) { + DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); + ip += inSeqs[idx].litLength; + seqPos->posInSrc += inSeqs[idx].litLength; + } + RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); + seqPos->idx = idx+1; + return 0; +} + +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch) +{ + U32 idx = seqPos->idx; + U32 startPosInSequence = seqPos->posInSequence; + U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; + size_t dictSize; + BYTE const* ip = (BYTE const*)(src); + BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + repcodes_t updatedRepcodes; + U32 bytesAdjustment = 0; + U32 finalMatchSplit = 0; + + /* TODO(embg) support fast parsing mode in noBlockDelim mode */ + (void)externalRepSearch; + + if (cctx->cdict) { + dictSize = cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { + const ZSTD_Sequence currSeq = inSeqs[idx]; + U32 litLength = currSeq.litLength; + U32 matchLength = currSeq.matchLength; + U32 const rawOffset = currSeq.offset; + U32 offBase; + + /* Modify the sequence depending on where endPosInSequence lies */ + if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { + if (startPosInSequence >= litLength) { + startPosInSequence -= litLength; + litLength = 0; + matchLength -= startPosInSequence; + } else { + litLength -= startPosInSequence; + } + /* Move to the next sequence */ + endPosInSequence -= currSeq.litLength + currSeq.matchLength; + startPosInSequence = 0; + } else { + /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence + does not reach the end of the match. So, we have to split the sequence */ + DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", + currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); + if (endPosInSequence > litLength) { + U32 firstHalfMatchLength; + litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; + firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; + if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { + /* Only ever split the match if it is larger than the block size */ + U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; + if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { + /* Move the endPosInSequence backward so that it creates match of minMatch length */ + endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + firstHalfMatchLength -= bytesAdjustment; + } + matchLength = firstHalfMatchLength; + /* Flag that we split the last match - after storing the sequence, exit the loop, + but keep the value of endPosInSequence */ + finalMatchSplit = 1; + } else { + /* Move the position in sequence backwards so that we don't split match, and break to store + * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence + * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so + * would cause the first half of the match to be too small + */ + bytesAdjustment = endPosInSequence - currSeq.litLength; + endPosInSequence = currSeq.litLength; + break; + } + } else { + /* This sequence ends inside the literals, break to store the last literals */ + break; + } + } + /* Check if this offset can be represented with a repcode */ + { U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, ZSTD_hasExtSeqProd(&cctx->appliedParams)), + "Sequence validation failed"); + } + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); + ip += matchLength + litLength; + if (!finalMatchSplit) + idx++; /* Next Sequence */ + } + DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); + seqPos->idx = idx; + seqPos->posInSequence = endPosInSequence; + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + iend -= bytesAdjustment; + if (ip != iend) { + /* Store any last literals */ + U32 lastLLSize = (U32)(iend - ip); + assert(ip <= iend); + DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); + seqPos->posInSrc += lastLLSize; + } + + return bytesAdjustment; +} + +typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); +static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) +{ + ZSTD_sequenceCopier sequenceCopier = NULL; + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; + } else if (mode == ZSTD_sf_noBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreNoBlockDelim; + } + assert(sequenceCopier != NULL); + return sequenceCopier; +} + +/* Discover the size of next block by searching for the delimiter. + * Note that a block delimiter **must** exist in this mode, + * otherwise it's an input error. + * The block size retrieved will be later compared to ensure it remains within bounds */ +static size_t +blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + int end = 0; + size_t blockSize = 0; + size_t spos = seqPos.idx; + DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); + assert(spos <= inSeqsSize); + while (spos < inSeqsSize) { + end = (inSeqs[spos].offset == 0); + blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; + if (end) { + if (inSeqs[spos].matchLength != 0) + RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); + break; + } + spos++; + } + if (!end) + RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); + return blockSize; +} + +/* More a "target" block size */ +static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining) +{ + int const lastBlock = (remaining <= blockSize); + return lastBlock ? remaining : blockSize; +} + +static size_t determine_blockSize(ZSTD_sequenceFormat_e mode, + size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); + if (mode == ZSTD_sf_noBlockDelimiters) + return blockSize_noDelimiter(blockSize, remaining); + { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); + FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); + if (explicitBlockSize > blockSize) + RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); + if (explicitBlockSize > remaining) + RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); + return explicitBlockSize; + } +} + +/* Compress, block-by-block, all of the sequences given. + * + * Returns the cumulative size of all compressed blocks (including their headers), + * otherwise a ZSTD error. + */ +static size_t +ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + size_t cSize = 0; + size_t remaining = srcSize; + ZSTD_sequencePosition seqPos = {0, 0, 0}; + + BYTE const* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t compressedSeqsSize; + size_t cBlockSize; + size_t additionalByteAdjustment; + size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, + cctx->blockSize, remaining, + inSeqs, inSeqsSize, seqPos); + U32 const lastBlock = (blockSize == remaining); + FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); + assert(blockSize <= remaining); + ZSTD_resetSeqStore(&cctx->seqStore); + DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize); + + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes); + FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); + blockSize -= additionalByteAdjustment; + + /* If blocks are too small, emit as a nocompress block */ + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE(ip, blockSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); + } + + DEBUGLOG(4, "cSize final total: %zu", cSize); + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + size_t compressedBlocksSize = 0; + size_t frameHeaderSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ + frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + XXH64_update(&cctx->xxhState, src, srcSize); + } + /* cSize includes block header size and compressed sequences size */ + compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(4, "Final compressed size: %zu", cSize); + return cSize; +} + +/*====== Finalize ======*/ + +static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) +{ + const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; + const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + return stableInput ? zcs->expectedInBuffer : nullInput; +} + +/*! ZSTD_flushStream() : + * @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + input.size = input.pos; /* do not ingest more input during flush */ + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); + if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); + return toFlush; + } +} + + +/*-===== Pre-defined compression levels =====-*/ +#include "clevels.h" + +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); + switch (cParams.strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } + return cParams; +} + +static int ZSTD_dedicatedDictSearch_isSupported( + ZSTD_compressionParameters const* cParams) +{ + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog > cParams->chainLog) + && (cParams->chainLog <= 24); +} + +/** + * Reverses the adjustment applied to cparams when enabling dedicated dict + * search. This is used to recover the params set to be used in the working + * context. (Otherwise, those tables would also grow.) + */ +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams) { + switch (cParams->strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + if (cParams->hashLog < ZSTD_HASHLOG_MIN) { + cParams->hashLog = ZSTD_HASHLOG_MIN; + } + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } +} + +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + } +} + +/*! ZSTD_getCParams_internal() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. + * Use dictSize == 0 for unknown or unused. + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); + int row; + DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); + + /* row */ + if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ + else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else row = compressionLevel; + + { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); + /* acceleration factor */ + if (compressionLevel < 0) { + int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); + cp.targetLength = (unsigned)(-clampedCompressionLevel); + } + /* refine parameters based on srcSize & dictSize */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); + } +} + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); + DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); + ZSTD_memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + params.fParams.contentSizeFlag = 1; + return params; +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +void ZSTD_registerSequenceProducer( + ZSTD_CCtx* zc, + void* extSeqProdState, + ZSTD_sequenceProducer_F extSeqProdFunc +) { + assert(zc != NULL); + ZSTD_CCtxParams_registerSequenceProducer( + &zc->requestedParams, extSeqProdState, extSeqProdFunc + ); +} + +void ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* extSeqProdState, + ZSTD_sequenceProducer_F extSeqProdFunc +) { + assert(params != NULL); + if (extSeqProdFunc != NULL) { + params->extSeqProdFunc = extSeqProdFunc; + params->extSeqProdState = extSeqProdState; + } else { + params->extSeqProdFunc = NULL; + params->extSeqProdState = NULL; + } +} diff --git a/tools/zstd/lib/compress/zstd_compress_internal.h b/tools/zstd/lib/compress/zstd_compress_internal.h new file mode 100644 index 00000000..e41d7b78 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_internal.h @@ -0,0 +1,1534 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" +#include "zstd_cwksp.h" +#ifdef ZSTD_MULTITHREAD +# include "zstdmt_compress.h" +#endif +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. + It's not a big deal though : candidate will just be sorted again. + Additionally, candidate position 1 will be lost. + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table reuse with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + + +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; + +typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; + +typedef struct { + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; + +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; + +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; + +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ +typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ + +typedef struct { + U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ + U32 len; /* Raw length of match */ +} ZSTD_match_t; + +typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { + int price; /* price from beginning of segment to this position */ + U32 off; /* offset of previous match */ + U32 mlen; /* length of previous match */ + U32 litlen; /* nb of literals since previous match */ + U32 rep[ZSTD_REP_NUM]; /* offset history after previous match */ +} ZSTD_optimal_t; + +typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; + +#define ZSTD_OPT_SIZE (ZSTD_OPT_NUM+3) +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_SIZE */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_SIZE */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_paramSwitch_e literalCompressionMode; +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ +} ZSTD_window_t; + +#define ZSTD_WINDOW_START_INDEX 2 + +typedef struct ZSTD_matchState_t ZSTD_matchState_t; + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + +struct ZSTD_matchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U64 hashSalt; /* For row-based matchFinder: salts the hash for reuse of tag table */ + U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */ + + U32* hashTable; + U32* hashTable3; + U32* chainTable; + + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ + optState_t opt; /* optimal parser state */ + const ZSTD_matchState_t* dictMatchState; + ZSTD_compressionParameters cParams; + const rawSeqStore_t* ldmSeqStore; + + /* Controls prefetching in some dictMatchState matchfinders. + * This behavior is controlled from the cctx ms. + * This parameter has no effect in the cdict ms. */ + int prefetchCDictTables; + + /* When == 0, lazy match finders insert every position. + * When != 0, lazy match finders only insert positions they search. + * This allows them to skip much faster over incompressible data, + * at a small cost to compression ratio. + */ + int lazySkipping; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_matchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + U32 loadedDictEnd; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; +} ldmState_t; + +typedef struct { + ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t +ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t +ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); + return (srcSize >> minlog) + 2; +} + +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_ps_enable: + return 0; + case ZSTD_ps_disable: + return 1; + default: + assert(0 /* impossible: pre-validated */); + ZSTD_FALLTHROUGH; + case ZSTD_ps_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void +ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) +{ + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + + +#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1) +#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2) +#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3) +#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */ +#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM) +#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM) +#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM) +#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM) +#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */ + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH + * Allowed to over-read literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeq(seqStore_t* seqStorePtr, + size_t litLength, const BYTE* literals, const BYTE* litLimit, + U32 offBase, + size_t matchLength) +{ + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offBase = offBase; + + /* match Length */ + assert(matchLength >= MINMATCH); + { size_t const mlBase = matchLength - MINMATCH; + if (mlBase>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_matchLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].mlBase = (U16)mlBase; + } + + seqStorePtr->sequences++; +} + +/* ZSTD_updateRep() : + * updates in-place @rep (array of repeat offsets) + * @offBase : sum-type, using numeric representation of ZSTD_storeSeq() + */ +MEM_STATIC void +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = OFFBASE_TO_OFFSET(offBase); + } else { /* repcode */ + U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } else { /* repCode == 0 */ + /* nothing to do */ + } + } +} + +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +MEM_STATIC repcodes_t +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + repcodes_t newReps; + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_updateRep(newReps.rep, offBase, ll0); + return newReps; +} + + +/*-************************************* +* Match length counter +***************************************/ +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); } + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); } +static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); } + + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt); + case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt); + case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt); + case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt); + case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt); + } +} + + +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* +* Round buffer management +***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif +/* Max current allowed */ +#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + ( ((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/** + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ + return window.dictLimit == ZSTD_WINDOW_START_INDEX && + window.lowLimit == ZSTD_WINDOW_START_INDEX && + (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; +} + +/** + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/** + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? + ZSTD_extDict : + ms->dictMatchState != NULL ? + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + ZSTD_noDict; +} + +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + +/** + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); + U32 const minIndexToOverflowCorrect = cycleSize + + MAX(maxDist, cycleSize) + + ZSTD_WINDOW_START_INDEX; + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + +/** + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, + void const* srcEnd) +{ + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } + return curr > ZSTD_CURRENT_MAX; +} + +/** + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, + U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const currentCycle = curr & cycleMask; + /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ + U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX + ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX) + : 0; + U32 const newCurrent = currentCycle + + currentCycleCorrection + + MAX(maxDist, cycleSize); + U32 const correction = curr - newCurrent; + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); + assert(curr > newCurrent); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } + + window->base += correction; + window->dictBase += correction; + if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->lowLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->lowLimit -= correction; + } + if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->dictLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->dictLimit -= correction; + } + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); + + ++window->nbOverflowCorrections; + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, + window->lowLimit); + return correction; +} + +/** + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. + * + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ +MEM_STATIC void +ZSTD_checkDictValidity(const ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + * + * We also have to invalidate the dictionary if ZSTD_window_update() has detected + * non-contiguous segments, which means that loadedDictEnd != window->dictLimit. + * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use + * dictMatchState, so setting it to NULL is not a problem. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } +} + +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + ZSTD_memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)" "; + window->dictBase = (BYTE const*)" "; + ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ + window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ + window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ + window->nbOverflowCorrections = 0; +} + +/** + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_window_update(ZSTD_window_t* window, + void const* src, size_t srcSize, + int forceNonContiguous) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); + /* Check if blocks follow each other */ + if (src != window->nextSrc || forceNonContiguous) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + /* ms->nextToUpdate = window->dictLimit; */ + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ( (ip+srcSize > window->dictBase + window->lowLimit) + & (ip < window->dictBase + window->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/** + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/** + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + + +/* debug functions */ +#if (DEBUGLEVEL>=2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u=0, sum=0; u<=max; u++) sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u=0; u<=max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", + u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + } +} + +#endif + +/* Short Cache */ + +/* Normally, zstd matchfinders follow this flow: + * 1. Compute hash at ip + * 2. Load index from hashTable[hash] + * 3. Check if *ip == *(base + index) + * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss. + * + * Short cache is an optimization which allows us to avoid step 3 most of the time + * when the data doesn't actually match. With short cache, the flow becomes: + * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip. + * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works. + * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue. + * + * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to + * dictMatchState matchfinders. + */ +#define ZSTD_SHORT_CACHE_TAG_BITS 8 +#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1) + +/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable. + * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */ +MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) { + size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK); + assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0); + hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag; +} + +/* Helper function for short cache matchfinders. + * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */ +MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) { + U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK; + U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK; + return tag1 == tag2; +} + +#if defined (__cplusplus) +} +#endif + +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(seqStore_t* ssPtr); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params); + + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small ( 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); + +/** ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + * Note that the block delimiter must include the last literals of the block. + */ +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + +/* Returns the number of bytes to move the current read position back by. + * Only non-zero if we ended up splitting a sequence. + * Otherwise, it may return a ZSTD error if something went wrong. + * + * This function will attempt to scan through blockSize bytes + * represented by the sequences in @inSeqs, + * storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + +/* Returns 1 if an external sequence producer is registered, otherwise returns 0. */ +MEM_STATIC int ZSTD_hasExtSeqProd(const ZSTD_CCtx_params* params) { + return params->extSeqProdFunc != NULL; +} + +/* =============================================================== + * Deprecated definitions that are still used internally to avoid + * deprecation warnings. These functions are exactly equivalent to + * their public variants, but avoid the deprecation warnings. + * =============================================================== */ + +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +#endif /* ZSTD_COMPRESS_H */ diff --git a/tools/zstd/lib/compress/zstd_compress_literals.c b/tools/zstd/lib/compress/zstd_compress_literals.c new file mode 100644 index 00000000..bfd4f11a --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_literals.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + + +/* ************************************************************** +* Debug Traces +****************************************************************/ +#if DEBUGLEVEL >= 2 + +static size_t showHexa(const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*)src; + size_t u; + for (u=0; u31) + (srcSize>4095); + + DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ZSTD_memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + return srcSize + flSize; +} + +static int allBytesIdentical(const void* src, size_t srcSize) +{ + assert(srcSize >= 1); + assert(src != NULL); + { const BYTE b = ((const BYTE*)src)[0]; + size_t p; + for (p=1; p31) + (srcSize>4095); + + assert(dstCapacity >= 4); (void)dstCapacity; + assert(allBytesIdentical(src, srcSize)); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1); + return flSize+1; +} + +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-(int)strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) +{ + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)", + disableLiteralCompression, (U32)srcSize, dstCapacity); + + DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const flags = 0 + | (bmi2 ? HUF_flags_bmi2 : 0) + | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0) + | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0) + | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0); + + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int); + huf_compress_f huf_compress; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; + cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize, + src, srcSize, + HUF_SYMBOLVALUE_MAX, LitHufLog, + entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, flags); + DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + DEBUGLOG(5, "reusing statistics from previous huffman block"); + hType = set_repeat; + } + } + + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } + if (cLitSize==1) { + /* A return value of 1 signals that the alphabet consists of a single symbol. + * However, in some rare circumstances, it could be the compressed size (a single byte). + * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`. + * (it's also necessary to not generate statistics). + * Therefore, in such a case, actively check that all bytes are identical. */ + if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); + return lhSize+cLitSize; +} diff --git a/tools/zstd/lib/compress/zstd_compress_literals.h b/tools/zstd/lib/compress/zstd_compress_literals.h new file mode 100644 index 00000000..b060c8ad --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_literals.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressRleLiteralsBlock() : + * Conditions : + * - All bytes in @src are identical + * - dstCapacity >= 4 */ +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/tools/zstd/lib/compress/zstd_compress_sequences.c b/tools/zstd/lib/compress/zstd_compress_sequences.c new file mode 100644 index 00000000..8872d4d3 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_sequences.c @@ -0,0 +1,442 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * compressibility. + */ + return nbSeq >= 2048; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + + assert(total > 0); + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or fewer symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); + *op = codeTable[0]; + return 1; + case set_repeat: + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); + assert(oend >= op); + { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offBase, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/tools/zstd/lib/compress/zstd_compress_sequences.h b/tools/zstd/lib/compress/zstd_compress_sequences.h new file mode 100644 index 00000000..4a3a05da --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_sequences.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "../common/fse.h" /* FSE_repeat, FSE_CTable */ +#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/tools/zstd/lib/compress/zstd_compress_superblock.c b/tools/zstd/lib/compress/zstd_compress_superblock.c new file mode 100644 index 00000000..628a2dcc --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_superblock.c @@ -0,0 +1,688 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_superblock.h" + +#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ +#include "hist.h" /* HIST_countFast_wksp */ +#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */ +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" + +/** ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if unable to compress. + * Or error code */ +static size_t +ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + { int const flags = bmi2 ? HUF_flags_bmi2 : 0; + const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags) + : HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return (size_t)(op-ostart); +} + +static size_t +ZSTD_seqDecompressedSize(seqStore_t const* seqStore, + const seqDef* sequences, size_t nbSeqs, + size_t litSize, int lastSubBlock) +{ + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + size_t n; + for (n=0; nllType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t +ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 128) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return (size_t)(op - ostart); + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return (size_t)(op - ostart); +} + +/** ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return (size_t)(op-ostart); +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + if (nbSeq == 0) return sequencesSectionHeaderSize; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +typedef struct { + size_t estLitSize; + size_t estBlockSize; +} EstimatedBlockSize; +static EstimatedBlockSize ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + EstimatedBlockSize ebs; + ebs.estLitSize = ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + ebs.estBlockSize = ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + ebs.estBlockSize += ebs.estLitSize + ZSTD_blockHeaderSize; + return ebs; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +static size_t countLiterals(seqStore_t const* seqStore, const seqDef* sp, size_t seqCount) +{ + size_t n, total = 0; + assert(sp != NULL); + for (n=0; n %zu bytes", seqCount, (const void*)sp, total); + return total; +} + +#define BYTESCALE 256 + +static size_t sizeBlockSequences(const seqDef* sp, size_t nbSeqs, + size_t targetBudget, size_t avgLitCost, size_t avgSeqCost, + int firstSubBlock) +{ + size_t n, budget = 0, inSize=0; + /* entropy headers */ + size_t const headerSize = (size_t)firstSubBlock * 120 * BYTESCALE; /* generous estimate */ + assert(firstSubBlock==0 || firstSubBlock==1); + budget += headerSize; + + /* first sequence => at least one sequence*/ + budget += sp[0].litLength * avgLitCost + avgSeqCost; + if (budget > targetBudget) return 1; + inSize = sp[0].litLength + (sp[0].mlBase+MINMATCH); + + /* loop over sequences */ + for (n=1; n targetBudget) + /* though continue to expand until the sub-block is deemed compressible */ + && (budget < inSize * BYTESCALE) ) + break; + } + + return n; +} + +/** ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written into the first block. + * The following blocks use repeat_mode to compress. + * Sub-blocks are all compressed, except the last one when beneficial. + * @return : compressed size of the super block (which features multiple ZSTD blocks) + * or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const seqDef* const sstart = seqStorePtr->sequencesStart; + const seqDef* const send = seqStorePtr->sequences; + const seqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */ + size_t const nbSeqs = (size_t)(send - sstart); + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + size_t const nbLiterals = (size_t)(lend - lstart); + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t const minTarget = ZSTD_TARGETCBLOCKSIZE_MIN; /* enforce minimum size, to reduce undesirable side effects */ + size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize); + int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed); + int writeSeqEntropy = 1; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (srcSize=%u, litSize=%u, nbSeq=%u)", + (unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart)); + + /* let's start by a general estimation for the full block */ + if (nbSeqs > 0) { + EstimatedBlockSize const ebs = + ZSTD_estimateSubBlockSize(lp, nbLiterals, + ofCodePtr, llCodePtr, mlCodePtr, nbSeqs, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, + writeLitEntropy, writeSeqEntropy); + /* quick estimation */ + size_t const avgLitCost = nbLiterals ? (ebs.estLitSize * BYTESCALE) / nbLiterals : BYTESCALE; + size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs; + const size_t nbSubBlocks = MAX((ebs.estBlockSize + (targetCBlockSize/2)) / targetCBlockSize, 1); + size_t n, avgBlockBudget, blockBudgetSupp=0; + avgBlockBudget = (ebs.estBlockSize * BYTESCALE) / nbSubBlocks; + DEBUGLOG(5, "estimated fullblock size=%u bytes ; avgLitCost=%.2f ; avgSeqCost=%.2f ; targetCBlockSize=%u, nbSubBlocks=%u ; avgBlockBudget=%.0f bytes", + (unsigned)ebs.estBlockSize, (double)avgLitCost/BYTESCALE, (double)avgSeqCost/BYTESCALE, + (unsigned)targetCBlockSize, (unsigned)nbSubBlocks, (double)avgBlockBudget/BYTESCALE); + /* simplification: if estimates states that the full superblock doesn't compress, just bail out immediately + * this will result in the production of a single uncompressed block covering @srcSize.*/ + if (ebs.estBlockSize > srcSize) return 0; + + /* compress and write sub-blocks */ + assert(nbSubBlocks>0); + for (n=0; n < nbSubBlocks-1; n++) { + /* determine nb of sequences for current sub-block + nbLiterals from next sequence */ + size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp), + avgBlockBudget + blockBudgetSupp, avgLitCost, avgSeqCost, n==0); + /* if reached last sequence : break to last sub-block (simplification) */ + assert(seqCount <= (size_t)(send-sp)); + if (sp + seqCount == send) break; + assert(seqCount > 0); + /* compress sub-block */ + { int litEntropyWritten = 0; + int seqEntropyWritten = 0; + size_t litSize = countLiterals(seqStorePtr, sp, seqCount); + const size_t decompressedSize = + ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 0); + size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + 0); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + + /* check compressibility, update state components */ + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes", + (unsigned)decompressedSize, (unsigned)cSize); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + sp += seqCount; + blockBudgetSupp = 0; + } } + /* otherwise : do not compress yet, coalesce current sub-block with following one */ + } + } /* if (nbSeqs > 0) */ + + /* write last block */ + DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp)); + { int litEntropyWritten = 0; + int seqEntropyWritten = 0; + size_t litSize = (size_t)(lend - lp); + size_t seqCount = (size_t)(send - sp); + const size_t decompressedSize = + ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 1); + size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, (size_t)(oend-op), + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + + /* update pointers, the nb of literals borrowed from next sequence must be preserved */ + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes", + (unsigned)decompressedSize, (unsigned)cSize); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + sp += seqCount; + } + } + + + if (writeLitEntropy) { + DEBUGLOG(5, "Literal entropy tables were never written"); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "Sequence entropy tables were never written => cancel, emit an uncompressed block"); + return 0; + } + + if (ip < iend) { + /* some data left : last part of the block sent uncompressed */ + size_t const rSize = (size_t)((iend - ip)); + size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock); + DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + const seqDef* seq; + repcodes_t rep; + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed all subBlocks: total compressed size = %u", + (unsigned)(op-ostart)); + return (size_t)(op-ostart); +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + unsigned lastBlock) +{ + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); +} diff --git a/tools/zstd/lib/compress/zstd_compress_superblock.h b/tools/zstd/lib/compress/zstd_compress_superblock.h new file mode 100644 index 00000000..8e494f0d --- /dev/null +++ b/tools/zstd/lib/compress/zstd_compress_superblock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +#include "../zstd.h" /* ZSTD_CCtx */ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ diff --git a/tools/zstd/lib/compress/zstd_cwksp.h b/tools/zstd/lib/compress/zstd_cwksp.h new file mode 100644 index 00000000..3eddbd33 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_cwksp.h @@ -0,0 +1,748 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_internal.h" +#include "../common/portability_macros.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + + +/* Set our tables and aligneds to align by 64 bytes */ +#define ZSTD_CWKSP_ALIGNMENT_BYTES 64 + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_aligned_init_once, + ZSTD_cwksp_alloc_aligned, + ZSTD_cwksp_alloc_buffers +} ZSTD_cwksp_alloc_phase_e; + +/** + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. + */ +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ->] free space [<- buffers][<- aligned][<- init once] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. These tables are 64-byte aligned. + * + * - Init once: these buffers require to be initialized at least once before + * use. They should be used when we want to skip memory initialization + * while not triggering memory checkers (like Valgrind) when reading from + * from this memory without writing to it first. + * These buffers should be used carefully as they might contain data + * from previous compressions. + * Buffers are aligned to 64 bytes. + * + * - Aligned: these buffers don't require any initialization before they're + * used. The user of the buffer should make sure they write into a buffer + * location before reading from it. + * Buffers are aligned to 64 bytes. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Init once / Tables + * 3. Aligned / Tables + * 4. Buffers / Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + void* initOnceStart; + + BYTE allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); + assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws)); + assert(ws->workspace <= ws->initOnceStart); +#if ZSTD_MEMORY_SANITIZER + { + intptr_t const offset = __msan_test_shadow(ws->initOnceStart, + (U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart); + (void)offset; +#if defined(ZSTD_MSAN_PRINT) + if(offset!=-1) { + __msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32); + } +#endif + assert(offset==-1); + }; +#endif +} + +/** + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + * + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size(). + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { + if (size == 0) + return 0; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +/** + * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. + * Used to determine the number of bytes required for a given "aligned". + */ +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES)); +} + +/** + * Returns the amount of additional space the cwksp must allocate + * for internal purposes (currently only alignment). + */ +MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { + /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES + * bytes to align the beginning of tables section and end of buffers; + */ + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + return slackSpace; +} + + +/** + * Return the number of additional bytes required to align a pointer to the given number of bytes. + * alignBytes must be a power of two. + */ +MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { + size_t const alignBytesMask = alignBytes - 1; + size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; + assert((alignBytes & alignBytesMask) == 0); + assert(bytes < alignBytes); + return bytes; +} + +/** + * Returns the initial value for allocStart which is used to determine the position from + * which we can allocate from the end of the workspace. + */ +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) { + return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1)); +} + +/** + * Internal function. Do not use directly. + * Reserves the given number of bytes within the aligned/buffer segment of the wksp, + * which counts from the end of the wksp (as opposed to the object/table segment). + * + * Returns a pointer to the beginning of that space. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) +{ + void* const alloc = (BYTE*)ws->allocStart - bytes; + void* const bottom = ws->tableEnd; + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + /* the area is reserved from the end of wksp. + * If it overlaps with tableValidEnd, it voids guarantees on values' range */ + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + return alloc; +} + +/** + * Moves the cwksp to the next phase, and does any necessary allocations. + * cwksp initialization must necessarily go through each phase in order. + * Returns a 0 on success, or zstd error + */ +MEM_STATIC size_t +ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) +{ + assert(phase >= ws->phase); + if (phase > ws->phase) { + /* Going from allocating objects to allocating initOnce / tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once && + phase >= ZSTD_cwksp_alloc_aligned_init_once) { + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + + { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ + void *const alloc = ws->objectEnd; + size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); + void *const objectEnd = (BYTE *) alloc + bytesToAlign; + DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); + RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, + "table phase - alignment initial allocation failed!"); + ws->objectEnd = objectEnd; + ws->tableEnd = objectEnd; /* table area starts being empty */ + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + } + } + ws->phase = phase; + ZSTD_cwksp_assert_internal_consistency(ws); + } + return 0; +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) +{ + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) +{ + void* alloc; + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) { + return NULL; + } + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + if (alloc) { + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + /* We need to keep the redzone poisoned while unpoisoning the bytes that + * are actually allocated. */ + __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE); + } + } +#endif + + return alloc; +} + +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) +{ + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + * This memory has been initialized at least once in the past. + * This doesn't mean it has been initialized this time, and it might contain data from previous + * operations. + * The main usage is for algorithms that might need read access into uninitialized memory. + * The algorithm must maintain safety under these conditions and must make sure it doesn't + * leak any of the past data (directly or in side channels). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + if(ptr && ptr < ws->initOnceStart) { + /* We assume the memory following the current allocation is either: + * 1. Not usable as initOnce memory (end of workspace) + * 2. Another initOnce buffer that has been allocated before (and so was previously memset) + * 3. An ASAN redzone, in which case we don't want to write on it + * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart. + * Note that we assume here that MSAN and ASAN cannot run in the same time. */ + ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes)); + ws->initOnceStart = ptr; + } +#if ZSTD_MEMORY_SANITIZER + assert(__msan_test_shadow(ptr, bytes) == -1); +#endif + return ptr; +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) +{ + void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return ptr; +} + +/** + * Aligned on 64 bytes. These buffers have the special property that + * their values remain constrained, allowing us to reuse them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) +{ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once; + void* alloc; + void* end; + void* top; + + /* We can only start allocating tables after we are done reserving space for objects at the + * start of the workspace */ + if(ws->phase < phase) { + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; + } + } + alloc = ws->tableEnd; + end = (BYTE *)alloc + bytes; + top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return alloc; +} + +/** + * Aligned on sizeof(void*). + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(4, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0); + assert(bytes % ZSTD_ALIGNOF(void*) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) +{ + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table reuse logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * Since tableValidEnd space and initOnce space may overlap we don't poison + * the initOnce portion as it break its promise. This means that this poisoning + * check isn't always applied fully. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + __msan_poison(ws->objectEnd, size); + } else { + assert(ws->initOnceStart >= ws->objectEnd); + __msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd); + } + } +#endif + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd)); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context reuse logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace except for the areas in which we expect memory reuse + * without initialization (objects, valid tables area and init once + * memory). */ + { + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } + } +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ZSTD_cwksp_initialAllocStart(ws); + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) { + ws->phase = ZSTD_cwksp_alloc_aligned_init_once; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_customMalloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) + if (ptr != NULL && customMem.customFree != NULL) { + __msan_unpoison(ptr, ZSTD_cwksp_sizeof(ws)); + } +#endif + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); +} + +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before it's used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +/* ZSTD_alignmentSpaceWithinBounds() : + * Returns if the estimated space needed for a wksp is within an acceptable limit of the + * actual amount of space used. + */ +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) { + /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice + * the alignment bytes difference between estimation and actual usage */ + return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) && + ZSTD_cwksp_used(ws) <= estimatedSpace; +} + + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CWKSP_H */ diff --git a/tools/zstd/lib/compress/zstd_double_fast.c b/tools/zstd/lib/compress/zstd_double_fast.c new file mode 100644 index 00000000..a4e9c50d --- /dev/null +++ b/tools/zstd/lib/compress/zstd_double_fast.c @@ -0,0 +1,770 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_double_fast.h" + +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) { + ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); + } + if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { + ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); + } + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) + hashSmall[smHash] = curr + i; + if (i == 0 || hashLarge[lgHash] == 0) + hashLarge[lgHash] = curr + i; + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); + } +} + + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); + + /* init */ + ip += ((ip - prefixLowest) == 0); + { + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = base + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = base + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check noDict repcode */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + if (idxl0 > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchl0) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } + } + + idxl1 = hashLong[hl1]; + matchl1 = base + idxl1; + + if (idxs0 > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(matchs0) == MEM_read32(ip)) { + goto _search_next_long; + } + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* check prefix long +1 match */ + if (idxl1 > prefixLowestIndex) { + if (MEM_read64(matchl1) == MEM_read64(ip1)) { + ip = ip1; + mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8; + offset = (U32)(ip-matchl1); + while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */ + goto _match_found; + } + } + + /* if no long +1 match, explore the short match we found */ + mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + offset = (U32)(ip - matchs0); + while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } +} + + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams; + const U32* const dictHashLong = dms->hashTable; + const U32* const dictHashSmall = dms->chainTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); + const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + + /* if a dictionary is attached, it must be within window range */ + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); + PREFETCH_AREA(dictHashLong, hashTableBytes); + PREFETCH_AREA(dictHashSmall, chainTableBytes); + } + + /* init */ + ip += (dictAndPrefixLength == 0); + + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; + U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); + int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); + U32 const curr = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + + /* check repcode */ + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + if (matchIndexL > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchLong) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictTagsMatchL) { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); + while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ + goto _match_found; + } } + + if (matchIndexS > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else if (dictTagsMatchS) { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } } + + ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif + continue; + +_search_next_long: + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = curr + 1; + + /* check prefix long +1 match */ + if (matchIndexL3 > prefixLowestIndex) { + if (MEM_read64(matchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictTagsMatchL3) { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ + goto _match_found; + } } } + + /* if no long +1 match, explore the short match we found */ + if (matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(curr - matchIndexS); + while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip - match); + while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + +_match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } /* while (ip < ilimit) */ + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_FN(noDict, 4) +ZSTD_GEN_DFAST_FN(noDict, 5) +ZSTD_GEN_DFAST_FN(noDict, 6) +ZSTD_GEN_DFAST_FN(noDict, 7) + +ZSTD_GEN_DFAST_FN(dictMatchState, 4) +ZSTD_GEN_DFAST_FN(dictMatchState, 5) +ZSTD_GEN_DFAST_FN(dictMatchState, 6) +ZSTD_GEN_DFAST_FN(dictMatchState, 7) + + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + } +} + + +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } +} + + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + U32 offset_1=rep[0], offset_2=rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + + if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ + & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + } else { + if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; + offset = curr - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + + } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = curr + 1; + if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; + ip++; + offset = curr+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + offset = curr - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + + } else { + ip += ((ip-anchor) >> kSearchStrength) + 1; + continue; + } } + + /* move to next sequence start */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ + & (offset_2 <= current2 - dictStartIndex)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +ZSTD_GEN_DFAST_FN(extDict, 4) +ZSTD_GEN_DFAST_FN(extDict, 5) +ZSTD_GEN_DFAST_FN(extDict, 6) +ZSTD_GEN_DFAST_FN(extDict, 7) + +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); + } +} + +#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ diff --git a/tools/zstd/lib/compress/zstd_double_fast.h b/tools/zstd/lib/compress/zstd_double_fast.h new file mode 100644 index 00000000..ce6ed8c9 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_double_fast.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST ZSTD_compressBlock_doubleFast +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE ZSTD_compressBlock_doubleFast_dictMatchState +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT ZSTD_compressBlock_doubleFast_extDict +#else +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST NULL +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_DOUBLEFAST_EXTDICT NULL +#endif /* ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/tools/zstd/lib/compress/zstd_fast.c b/tools/zstd/lib/compress/zstd_fast.c new file mode 100644 index 00000000..6c4554cf --- /dev/null +++ b/tools/zstd/lib/compress/zstd_fast.c @@ -0,0 +1,968 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ +#include "zstd_fast.h" + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_full); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } + + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); + } } } } +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_fast); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = curr; + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = curr + p; + } } } } +} + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillHashTableForCCtx(ms, end, dtlm); + } +} + + +/** + * If you squint hard enough (and ignore repcodes), the search operation at any + * given position is broken into 4 stages: + * + * 1. Hash (map position to hash value via input read) + * 2. Lookup (map hash val to index via hashtable read) + * 3. Load (map index to value at that position via input read) + * 4. Compare + * + * Each of these steps involves a memory read at an address which is computed + * from the previous step. This means these steps must be sequenced and their + * latencies are cumulative. + * + * Rather than do 1->2->3->4 sequentially for a single position before moving + * onto the next, this implementation interleaves these operations across the + * next few positions: + * + * R = Repcode Read & Compare + * H = Hash + * T = Table Lookup + * M = Match Read & Compare + * + * Pos | Time --> + * ----+------------------- + * N | ... M + * N+1 | ... TM + * N+2 | R H T M + * N+3 | H TM + * N+4 | R H T M + * N+5 | H ... + * N+6 | R ... + * + * This is very much analogous to the pipelining of execution in a CPU. And just + * like a CPU, we have to dump the pipeline when we find a match (i.e., take a + * branch). + * + * When this happens, we throw away our current state, and do the following prep + * to re-enter the loop: + * + * Pos | Time --> + * ----+------------------- + * N | H T + * N+1 | H + * + * This is also the work we do at the beginning to enter the loop initially. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + + const BYTE* anchor = istart; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + U32 rep_offset1 = rep[0]; + U32 rep_offset2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + U32 mval; /* src value at match idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + + /* ip0 and ip1 are always adjacent. The targetLength skipping and + * uncompressibility acceleration is applied to every other position, + * matching the behavior of #1562. step therefore represents the gap + * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); + ip0 += (ip0 == prefixStart); + { U32 const curr = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; + if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + + do { + /* load repcode match for ip[2]*/ + const U32 rval = MEM_read32(ip2 - rep_offset1); + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { + ip0 = ip2; + match0 = ip0 - rep_offset1; + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 is before the + * repcode (ip2). */ + hashTable[hash1] = (U32)(ip1 - base); + + goto _match; + } + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 == ip0 + 1, so + * we know we will resume searching after ip1 */ + hashTable[hash1] = (U32)(ip1 - base); + + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + + /* first write next hash table entry; we've already calculated it */ + if (step <= 4) { + /* We need to avoid writing an index into the hash table >= the + * position at which we will pick up our searching after we've + * taken this match. + * + * The minimum possible match has length 4, so the earliest ip0 + * can be after we take this match will be the current ip0 + 4. + * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely + * write this position. + */ + hashTable[hash1] = (U32)(ip1 - base); + } + + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* When the repcodes are outside of the prefix, we set them to zero before the loop. + * When the offsets are still zero, we need to restore them after the block to have a correct + * repcode history. If only one offset was invalid, it is easy. The tricky case is when both + * offsets were invalid. We need to figure out which offset to refill with. + * - If both offsets are zero they are in the same order. + * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. + * - If only one is zero, we need to decide which offset to restore. + * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. + * - It is impossible for rep_offset2 to be non-zero. + * + * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then + * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. + */ + offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx */ + + /* Compute the offset code. */ + match0 = base + idx; + rep_offset2 = rep_offset1; + rep_offset1 = (U32)(ip0-match0); + offcode = OFFSET_TO_OFFBASE(rep_offset1); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } + +_match: /* Requires: ip0, match0, offcode */ + + /* Count the forward length. */ + mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; + { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + + goto _start; +} + +#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \ + static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \ + } + +ZSTD_GEN_FAST_FN(noDict, 4, 1) +ZSTD_GEN_FAST_FN(noDict, 5, 1) +ZSTD_GEN_FAST_FN(noDict, 6, 1) +ZSTD_GEN_FAST_FN(noDict, 7, 1) + +ZSTD_GEN_FAST_FN(noDict, 4, 0) +ZSTD_GEN_FAST_FN(noDict, 5, 0) +ZSTD_GEN_FAST_FN(noDict, 6, 0) +ZSTD_GEN_FAST_FN(noDict, 7, 0) + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + if (ms->cParams.targetLength > 1) { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } + + } +} + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip0 = istart; + const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; + const U32* const dictHashTable = dms->hashTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); + const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); + const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + /* ensure there will be no underflow + * when translating a dict index into a local index */ + assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + PREFETCH_AREA(dictHashTable, hashTableBytes); + } + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); + ip0 += (dictAndPrefixLength == 0); + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Outer search loop */ + assert(stepSize >= 1); + while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ + size_t mLength; + size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); + + size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); + U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); + + U32 matchIndex = hashTable[hash0]; + U32 curr = (U32)(ip0 - base); + size_t step = stepSize; + const size_t kStepIncr = 1 << kSearchStrength; + const BYTE* nextStep = ip0 + kStepIncr; + + /* Inner search loop */ + while (1) { + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); + size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); + hashTable[hash0] = curr; /* update hash table */ + + if (((U32) ((prefixStartIndex - 1) - repIndex) >= + 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip0++; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + break; + } + + if (dictTagsMatch) { + /* Found a possible dict match */ + const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex > dictStartIndex && + MEM_read32(dictMatch) == MEM_read32(ip0)) { + /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ + if (matchIndex <= prefixStartIndex) { + U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip0 > anchor) & (dictMatch > dictStart)) + && (ip0[-1] == dictMatch[-1])) { + ip0--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + } + } + + if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) { + /* found a regular match */ + U32 const offset = (U32) (ip0 - match); + mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; + while (((ip0 > anchor) & (match > prefixStart)) + && (ip0[-1] == match[-1])) { + ip0--; + match--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + + /* Prepare for next iteration */ + dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; + dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); + matchIndex = hashTable[hash1]; + + if (ip1 >= nextStep) { + step++; + nextStep += kStepIncr; + } + ip0 = ip1; + ip1 = ip1 + step; + if (ip1 > ilimit) goto _cleanup; + + curr = (U32)(ip0 - base); + hash0 = hash1; + } /* end inner search loop */ + + /* match found */ + assert(mLength); + ip0 += mLength; + anchor = ip0; + + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+curr+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + /* check immediate repcode */ + while (ip0 <= ilimit) { + U32 const current2 = (U32)(ip0-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip0))) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } + } + + /* Prepare for next iteration */ + assert(ip0 == anchor); + ip1 = ip0 + stepSize; + } + +_cleanup: + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) + +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState != NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); + } +} + + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const BYTE* const dictStart = dictBase + dictStartIndex; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + const BYTE* idxBase; /* base pointer for idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ + + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); + + { U32 const curr = (U32)(ip0 - base); + U32 const maxRep = curr - dictStartIndex; + if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + do { + { /* load repcode match for ip[2] */ + U32 const current2 = (U32)(ip2 - base); + U32 const repIndex = current2 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + U32 rval; + if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ + & (offset_1 > 0) ) { + rval = MEM_read32(repBase + repIndex); + } else { + rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ + } + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if (MEM_read32(ip2) == rval) { + ip0 = ip2; + match0 = repBase + repIndex; + matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + assert((match0 != prefixStart) & (match0 != dictStart)); + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + goto _match; + } } + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx, idxBase */ + + /* Compute the offset code. */ + { U32 const offset = current0 - idx; + const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; + matchEnd = idx < prefixStartIndex ? dictEnd : iend; + match0 = idxBase + idx; + offset_2 = offset_1; + offset_1 = offset; + offcode = OFFSET_TO_OFFBASE(offset); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } } + +_match: /* Requires: ip0, match0, offcode, matchEnd */ + + /* Count the forward length. */ + assert(matchEnd != 0); + mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while (ip0 <= ilimit) { + U32 const repIndex2 = (U32)(ip0-base) - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } } + + goto _start; +} + +ZSTD_GEN_FAST_FN(extDict, 4, 0) +ZSTD_GEN_FAST_FN(extDict, 5, 0) +ZSTD_GEN_FAST_FN(extDict, 6, 0) +ZSTD_GEN_FAST_FN(extDict, 7, 0) + +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); + } +} diff --git a/tools/zstd/lib/compress/zstd_fast.h b/tools/zstd/lib/compress/zstd_fast.h new file mode 100644 index 00000000..9e4236b4 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_fast.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/tools/zstd/lib/compress/zstd_lazy.c b/tools/zstd/lib/compress/zstd_lazy.c new file mode 100644 index 00000000..67dd55fd --- /dev/null +++ b/tools/zstd/lib/compress/zstd_lazy.c @@ -0,0 +1,2199 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_lazy.h" +#include "../common/bits.h" /* ZSTD_countTrailingZeros64 */ + +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) + +#define kLazySkippingStep 8 + + +/*-************************************* +* Binary Tree search +***************************************/ + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_updateDUBT(ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iend, + U32 mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; + + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for ( ; idx < target ; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; + + U32* const nextCandidatePtr = bt + 2*(idx&btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; + + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; + } + ms->nextToUpdate = target; +} + + +/** ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : curr >= btlow == (curr - btmask) + * doesn't fail */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_insertDUBT1(const ZSTD_matchState_t* ms, + U32 curr, const BYTE* inputEnd, + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; + + + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + curr, dictLimit, windowLow); + assert(curr >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ + + if ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ + || (curr < dictLimit) /* both in extDict */) { + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; + assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (curr < dictLimit) ); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ + } + + DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", + curr, matchIndex, (U32)matchLength); + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", + matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", + matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; +} + + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_DUBT_findBetterDictMatch ( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + size_t bestLength, + U32 nbCompares, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_matchState_t * const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32 * const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; + + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const curr = (U32)(ip-base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; + + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + + size_t commonLengthSmaller=0, commonLengthLarger=0; + + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); + + for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) { + U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex+matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + } + if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + +} + + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offBasePtr, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); + + U32* nextCandidate = bt + 2*(matchIndex&btMask); + U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; + + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); + assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); + + /* reach end of unsorted candidates list */ + while ( (matchIndex > unsortLimit) + && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) + && (nbCandidates > 1) ) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", + matchIndex); + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2*(matchIndex&btMask); + unsortedMark = bt + 2*(matchIndex&btMask) + 1; + nbCandidates --; + } + + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ( (matchIndex > unsortLimit) + && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", + matchIndex); + *nextCandidate = *unsortedMark = 0; + } + + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, + nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } + + /* find longest match */ + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + matchIndex = hashTable[h]; + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) ) + bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch( + ms, ip, iend, + offBasePtr, bestLength, nbCompares, + mls, dictMode); + } + + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offBasePtr, mIndex); + } + return bestLength; + } +} + + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offBasePtr, + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode); +} + +/*********************************** +* Dedicated dict search +***********************************/ + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target - idx ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; + + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + U32 hashIdx; + + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog > ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); + + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || ++countBeyondMinChain > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } + + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } + + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } + + ms->nextToUpdate = target; +} + +/* Returns the longest match length found in the dedicated dict search structure. + * If none are longer than the argument ml, then ml will be returned. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, + const ZSTD_matchState_t* const dms, + const BYTE* const ip, const BYTE* const iLimit, + const BYTE* const prefixStart, const U32 curr, + const U32 dictLimit, const size_t ddsIdx) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + U32 matchIndex; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + return ml; +} + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls, U32 const lazySkipping) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + /* Stop inserting every position when in the lazy skipping mode. */ + if (lazySkipping) + break; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0); +} + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_HcFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml=4-1; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + + U32 matchIndex; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } + + /* HC4 match finder */ + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping); + + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) break; + + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } + } + + return ml; +} + +/* ********************************* +* (SIMD) Row-based matchfinder +***********************************/ +/* Constants for row-based hash */ +#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) +#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ + +#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1) + +typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */ + +/* ZSTD_VecMask_next(): + * Starting from the LSB, returns the idx of the next non-zero bit. + * Basically counting the nb of trailing zeroes. + */ +MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + return ZSTD_countTrailingZeros64(val); +} + +/* ZSTD_row_nextIndex(): + * Returns the next index to insert at within a tagTable row, and updates the "head" + * value to reflect the update. Essentially cycles backwards from [1, {entries per row}) + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { + U32 next = (*tagRow-1) & rowMask; + next += (next == 0) ? rowMask : 0; /* skip first position */ + *tagRow = (BYTE)next; + return next; +} + +/* ZSTD_isAligned(): + * Checks that a pointer is aligned to "align" bytes which must be a power of 2. + */ +MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { + assert((align & (align - 1)) == 0); + return (((size_t)ptr) & (align - 1)) == 0; +} + +/* ZSTD_row_prefetch(): + * Performs prefetching for the hashTable and tagTable at a given row. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) { + PREFETCH_L1(hashTable + relRow); + if (rowLog >= 5) { + PREFETCH_L1(hashTable + relRow + 16); + /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */ + } + PREFETCH_L1(tagTable + relRow); + if (rowLog == 6) { + PREFETCH_L1(tagTable + relRow + 32); + } + assert(rowLog == 4 || rowLog == 5 || rowLog == 6); + assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */ + assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */ +} + +/* ZSTD_row_fillHashCache(): + * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries, + * but not beyond iLimit. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base, + U32 const rowLog, U32 const mls, + U32 idx, const BYTE* const iLimit) +{ + U32 const* const hashTable = ms->hashTable; + BYTE const* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); + U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); + + for (; idx < lim; ++idx) { + U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; + } + + DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1], + ms->hashCache[2], ms->hashCache[3], ms->hashCache[4], + ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]); +} + +/* ZSTD_row_nextCachedHash(): + * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at + * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, + BYTE const* tagTable, BYTE const* base, + U32 idx, U32 const hashLog, + U32 const rowLog, U32 const mls, + U64 const hashSalt) +{ + U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; + cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash; + return hash; + } +} + +/* ZSTD_row_update_internalImpl(): + * Updates the hash table with positions starting from updateStartIdx until updateEndIdx. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms, + U32 updateStartIdx, U32 const updateEndIdx, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + const BYTE* const base = ms->window.base; + + DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); + for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt) + : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32* const row = hashTable + relRow; + BYTE* tagRow = tagTable + relRow; + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + + assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt)); + tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK; + row[pos] = updateStartIdx; + } +} + +/* ZSTD_row_update_internal(): + * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate. + * Skips sections of long matches as is necessary. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32 idx = ms->nextToUpdate; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + const U32 kSkipThreshold = 384; + const U32 kMaxMatchStartPositionsToUpdate = 96; + const U32 kMaxMatchEndPositionsToUpdate = 32; + + if (useCache) { + /* Only skip positions when using hash cache, i.e. + * if we are loading a dict, don't skip anything. + * If we decide to skip, then we only update a set number + * of positions at the beginning and end of the match. + */ + if (UNLIKELY(target - idx > kSkipThreshold)) { + U32 const bound = idx + kMaxMatchStartPositionsToUpdate; + ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache); + idx = target - kMaxMatchEndPositionsToUpdate; + ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1); + } + } + assert(target >= idx); + ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache); + ms->nextToUpdate = target; +} + +/* ZSTD_row_update(): + * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary + * processing. + */ +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) { + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + const U32 rowMask = (1u << rowLog) - 1; + const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); + + DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */); +} + +/* Returns the mask width of bits group of which will be set to 1. Given not all + * architectures have easy movemask instruction, this helps to iterate over + * groups of bits easier and faster. + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_row_matchMaskGroupWidth(const U32 rowEntries) +{ + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + (void)rowEntries; +#if defined(ZSTD_ARCH_ARM_NEON) + /* NEON path only works for little endian */ + if (!MEM_isLittleEndian()) { + return 1; + } + if (rowEntries == 16) { + return 4; + } + if (rowEntries == 32) { + return 2; + } + if (rowEntries == 64) { + return 1; + } +#endif + return 1; +} + +#if defined(ZSTD_ARCH_X86_SSE2) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head) +{ + const __m128i comparisonMask = _mm_set1_epi8((char)tag); + int matches[4] = {0}; + int i; + assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4); + for (i=0; i> chunkSize; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= (chunk * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } else { /* big endian: reverse bits during extraction */ + const size_t msb = xFF ^ (xFF >> 1); + const size_t extractMagic = (msb / 0x1FF) | msb; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= ((chunk >> 7) * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } + matches = ~matches; + if (rowEntries == 16) { + return ZSTD_rotateRight_U16((U16)matches, headGrouped); + } else if (rowEntries == 32) { + return ZSTD_rotateRight_U32((U32)matches, headGrouped); + } else { + return ZSTD_rotateRight_U64((U64)matches, headGrouped); + } + } +#endif +} + +/* The high-level approach of the SIMD row based match finder is as follows: + * - Figure out where to insert the new entry: + * - Generate a hash for current input posistion and split it into a one byte of tag and `rowHashLog` bits of index. + * - The hash is salted by a value that changes on every contex reset, so when the same table is used + * we will avoid collisions that would otherwise slow us down by intorducing phantom matches. + * - The hashTable is effectively split into groups or "rows" of 15 or 31 entries of U32, and the index determines + * which row to insert into. + * - Determine the correct position within the row to insert the entry into. Each row of 15 or 31 can + * be considered as a circular buffer with a "head" index that resides in the tagTable (overall 16 or 32 bytes + * per row). + * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte tag calculated for the position and + * generate a bitfield that we can cycle through to check the collisions in the hash table. + * - Pick the longest match. + * - Insert the tag into the equivalent row and position in the tagTable. + */ +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_RowFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode, + const U32 rowLog) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32* const hashCache = ms->hashCache; + const U32 hashLog = ms->rowHashLog; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 rowEntries = (1U << rowLog); + const U32 rowMask = rowEntries - 1; + const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries); + const U64 hashSalt = ms->hashSalt; + U32 nbAttempts = 1U << cappedSearchLog; + size_t ml=4-1; + U32 hash; + + /* DMS/DDS variables that may be referenced laster */ + const ZSTD_matchState_t* const dms = ms->dictMatchState; + + /* Initialize the following variables to satisfy static analyzer */ + size_t ddsIdx = 0; + U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */ + U32 dmsTag = 0; + U32* dmsRow = NULL; + BYTE* dmsTagRow = NULL; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + { /* Prefetch DDS hashtable entry */ + ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG; + PREFETCH_L1(&dms->hashTable[ddsIdx]); + } + ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0; + } + + if (dictMode == ZSTD_dictMatchState) { + /* Prefetch DMS rows */ + U32* const dmsHashTable = dms->hashTable; + BYTE* const dmsTagTable = dms->tagTable; + U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; + dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow); + dmsRow = dmsHashTable + dmsRelRow; + ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog); + } + + /* Update the hashTable and tagTable up to (but not including) ip */ + if (!ms->lazySkipping) { + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt); + } else { + /* Stop inserting every position when in the lazy skipping mode. + * The hash cache is also not kept up to date in this mode. + */ + hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + ms->nextToUpdate = curr; + } + ms->hashSaltEntropy += hash; /* collect salt entropy */ + + { /* Get the hash for ip, compute the appropriate row */ + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); + U32 const headGrouped = (*tagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries); + + /* Cycle through the matches and prefetch */ + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = row[matchPos]; + if(matchPos == 0) continue; + assert(numMatches < rowEntries); + if (matchIndex < lowLimit) + break; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + PREFETCH_L1(base + matchIndex); + } else { + PREFETCH_L1(dictBase + matchIndex); + } + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop + in ZSTD_row_update_internal() at the next search. */ + { + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + tagRow[pos] = (BYTE)tag; + row[pos] = ms->nextToUpdate++; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex < curr); + assert(matchIndex >= lowLimit); + + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* Save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + /* TODO: Measure and potentially add prefetching to DMS */ + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + + { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries); + + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = dmsRow[matchPos]; + if(matchPos == 0) continue; + if (matchIndex < dmsLowestIndex) + break; + PREFETCH_L1(dmsBase + matchIndex); + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex >= dmsLowestIndex); + assert(matchIndex < curr); + + { const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + } + + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; + } + } + } + } + return ml; +} + + +/** + * Generate search functions templated on (dictMode, mls, rowLog). + * These functions are outlined for code size & compilation time. + * ZSTD_searchMax() dispatches to the correct implementation function. + * + * TODO: The start of the search function involves loading and calculating a + * bunch of constants from the ZSTD_matchState_t. These computations could be + * done in an initialization function, and saved somewhere in the match state. + * Then we could pass a pointer to the saved state instead of the match state, + * and avoid duplicate computations. + * + * TODO: Move the match re-winding into searchMax. This improves compression + * ratio, and unlocks further simplifications with the next TODO. + * + * TODO: Try moving the repcode search into searchMax. After the re-winding + * and repcode search are in searchMax, there is no more logic in the match + * finder loop that requires knowledge about the dictMode. So we should be + * able to avoid force inlining it, and we can join the extDict loop with + * the single segment loop. It should go in searchMax instead of its own + * function to avoid having multiple virtual function calls per search. + */ + +#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls +#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls +#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog + +#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE + +#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offBasePtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ + return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ + } \ + +#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ + X(dictMode, mls, 4) \ + X(dictMode, mls, 5) \ + X(dictMode, mls, 6) + +#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6) + +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate row search fns for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN) +/* Generate binary Tree search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN) +/* Generate hash chain search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN) + +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + case rowLog: \ + return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr); + +#define ZSTD_SWITCH_MLS(X, dictMode) \ + switch (mls) { \ + ZSTD_FOR_EACH_MLS(X, dictMode) \ + } + +#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \ + case mls: \ + switch (rowLog) { \ + ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \ + } \ + ZSTD_UNREACHABLE; \ + break; + +#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \ + switch (searchMethod) { \ + case search_hashChain: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \ + break; \ + case search_binaryTree: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \ + break; \ + case search_rowHash: \ + ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \ + break; \ + } \ + ZSTD_UNREACHABLE; + +/** + * Searches for the longest match at @p ip. + * Dispatches to the correct implementation function based on the + * (searchMethod, dictMode, mls, rowLog). We use switch statements + * here instead of using an indirect function call through a function + * pointer because after Spectre and Meltdown mitigations, indirect + * function calls can be very costly, especially in the kernel. + * + * NOTE: dictMode and searchMethod should be templated, so those switch + * statements should be optimized out. Only the mls & rowLog switches + * should be left. + * + * @param ms The match state. + * @param ip The position to search at. + * @param iend The end of the input data. + * @param[out] offsetPtr Stores the match offset into this pointer. + * @param mls The minimum search length, in the range [4, 6]. + * @param rowLog The row log (if applicable), in the range [4, 6]. + * @param searchMethod The search method to use (templated). + * @param dictMode The dictMode (templated). + * + * @returns The length of the longest match found, or < mls if no match is found. + * If a match is found its offset is stored in @p offsetPtr. + */ +FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( + ZSTD_matchState_t* ms, + const BYTE* ip, + const BYTE* iend, + size_t* offsetPtr, + U32 const mls, + U32 const rowLog, + searchMethod_e const searchMethod, + ZSTD_dictMode_e const dictMode) +{ + if (dictMode == ZSTD_noDict) { + ZSTD_SWITCH_SEARCH_METHOD(noDict) + } else if (dictMode == ZSTD_extDict) { + ZSTD_SWITCH_SEARCH_METHOD(extDict) + } else if (dictMode == ZSTD_dictMatchState) { + ZSTD_SWITCH_SEARCH_METHOD(dictMatchState) + } else if (dictMode == ZSTD_dedicatedDictSearch) { + ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch) + } + ZSTD_UNREACHABLE; + return 0; +} + +/* ******************************* +* Common parser - lazy strategy +*********************************/ + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + if (isDxS) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + DEBUGLOG(7, "search baseline (depth 0)"); + + /* check repCode */ + if (isDxS) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } + } + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offbaseFound = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = offbaseFound; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */; + ip += step; + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * Pay attention that `start[-value]` can lead to strange undefined behavior + * notably if `value` is unsigned, resulting in a large positive `-value`. + */ + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest)) + && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (isDxS) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } + + /* check immediate repcode */ + if (isDxS) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} +#endif /* build exclusions */ + + +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} +#endif + +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + /* init */ + ip += (ip == prefixStart); + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + U32 curr = (U32)(ip-base); + + /* check repCode */ + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = ofbCandidate; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength); + ip += step + 1; /* jump faster over incompressible sections */ + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } + + /* search match, depth 1 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } + + /* search match, depth 2 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} +#endif /* build exclusions */ + +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); +} + +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); +} + +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1); +} +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); +} + +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +} +#endif diff --git a/tools/zstd/lib/compress/zstd_lazy.h b/tools/zstd/lib/compress/zstd_lazy.h new file mode 100644 index 00000000..3635813b --- /dev/null +++ b/tools/zstd/lib/compress/zstd_lazy.h @@ -0,0 +1,202 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/** + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ + +#if !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); + +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ +#endif + +#ifndef ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_GREEDY ZSTD_compressBlock_greedy +#define ZSTD_COMPRESSBLOCK_GREEDY_ROW ZSTD_compressBlock_greedy_row +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE ZSTD_compressBlock_greedy_dictMatchState +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW ZSTD_compressBlock_greedy_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH ZSTD_compressBlock_greedy_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_greedy_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT ZSTD_compressBlock_greedy_extDict +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW ZSTD_compressBlock_greedy_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_GREEDY NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_GREEDY_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_LAZY ZSTD_compressBlock_lazy +#define ZSTD_COMPRESSBLOCK_LAZY_ROW ZSTD_compressBlock_lazy_row +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE ZSTD_compressBlock_lazy_dictMatchState +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT ZSTD_compressBlock_lazy_extDict +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW ZSTD_compressBlock_lazy_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_LAZY NULL +#define ZSTD_COMPRESSBLOCK_LAZY_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_LAZY_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_LAZY_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_LAZY2 ZSTD_compressBlock_lazy2 +#define ZSTD_COMPRESSBLOCK_LAZY2_ROW ZSTD_compressBlock_lazy2_row +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE ZSTD_compressBlock_lazy2_dictMatchState +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW ZSTD_compressBlock_lazy2_dictMatchState_row +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH ZSTD_compressBlock_lazy2_dedicatedDictSearch +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW ZSTD_compressBlock_lazy2_dedicatedDictSearch_row +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT ZSTD_compressBlock_lazy2_extDict +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW ZSTD_compressBlock_lazy2_extDict_row +#else +#define ZSTD_COMPRESSBLOCK_LAZY2 NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DICTMATCHSTATE_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_DEDICATEDDICTSEARCH_ROW NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_LAZY2_EXTDICT_ROW NULL +#endif + +#ifndef ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTLAZY2 ZSTD_compressBlock_btlazy2 +#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE ZSTD_compressBlock_btlazy2_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT ZSTD_compressBlock_btlazy2_extDict +#else +#define ZSTD_COMPRESSBLOCK_BTLAZY2 NULL +#define ZSTD_COMPRESSBLOCK_BTLAZY2_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTLAZY2_EXTDICT NULL +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LAZY_H */ diff --git a/tools/zstd/lib/compress/zstd_ldm.c b/tools/zstd/lib/compress/zstd_ldm.c new file mode 100644 index 00000000..17c069fe --- /dev/null +++ b/tools/zstd/lib/compress/zstd_ldm.c @@ -0,0 +1,730 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_ldm.h" + +#include "../common/debug.h" +#include "../common/xxhash.h" +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ +#include "zstd_ldm_geartab.h" + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 + +typedef struct { + U64 rolling; + U64 stopMask; +} ldmRollingHashState_t; + +/** ZSTD_ldm_gear_init(): + * + * Initializes the rolling hash state such that it will honor the + * settings in params. */ +static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) +{ + unsigned maxBitsInMask = MIN(params->minMatchLength, 64); + unsigned hashRateLog = params->hashRateLog; + + state->rolling = ~(U32)0; + + /* The choice of the splitting criterion is subject to two conditions: + * 1. it has to trigger on average every 2^(hashRateLog) bytes; + * 2. ideally, it has to depend on a window of minMatchLength bytes. + * + * In the gear hash algorithm, bit n depends on the last n bytes; + * so in order to obtain a good quality splitting criterion it is + * preferable to use bits with high weight. + * + * To match condition 1 we use a mask with hashRateLog bits set + * and, because of the previous remark, we make sure these bits + * have the highest possible weight while still respecting + * condition 2. + */ + if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { + state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); + } else { + /* In this degenerate case we simply honor the hash rate. */ + state->stopMask = ((U64)1 << hashRateLog) - 1; + } +} + +/** ZSTD_ldm_gear_reset() + * Feeds [data, data + minMatchLength) into the hash without registering any + * splits. This effectively resets the hash state. This is used when skipping + * over data, either at the beginning of a block, or skipping sections. + */ +static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state, + BYTE const* data, size_t minMatchLength) +{ + U64 hash = state->rolling; + size_t n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + } while (0) + while (n + 3 < minMatchLength) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < minMatchLength) { + GEAR_ITER_ONCE(); + } +#undef GEAR_ITER_ONCE +} + +/** ZSTD_ldm_gear_feed(): + * + * Registers in the splits array all the split points found in the first + * size bytes following the data pointer. This function terminates when + * either all the data has been processed or LDM_BATCH_SIZE splits are + * present in the splits array. + * + * Precondition: The splits array must not be full. + * Returns: The number of bytes processed. */ +static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, + BYTE const* data, size_t size, + size_t* splits, unsigned* numSplits) +{ + size_t n; + U64 hash, mask; + + hash = state->rolling; + mask = state->stopMask; + n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + if (UNLIKELY((hash & mask) == 0)) { \ + splits[*numSplits] = n; \ + *numSplits += 1; \ + if (*numSplits == LDM_BATCH_SIZE) \ + goto done; \ + } \ + } while (0) + + while (n + 3 < size) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < size) { + GEAR_ITER_ONCE(); + } + +#undef GEAR_ITER_ONCE + +done: + state->rolling = hash; + return n; +} + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams) +{ + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog + ? 0 + : params->windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); + return params.enableLdm == ZSTD_ps_enable ? totalSize : 0; +} + +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) +{ + return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0; +} + +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + ldmParams_t const ldmParams) +{ + BYTE* const pOffset = ldmState->bucketOffsets + hash; + unsigned const offset = *pOffset; + + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + +} + +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} + +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, + void const* end) +{ + const BYTE* const iend = (const BYTE*)end; + + switch(ms->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); + break; + + case ZSTD_dfast: +#ifndef ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); +#else + assert(0); /* shouldn't be called: cparams should've been adjusted. */ +#endif + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +void ZSTD_ldm_fillHashTable( + ldmState_t* ldmState, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + U32 const minMatchLength = params->minMatchLength; + U32 const hBits = params->hashLog - params->bucketSizeLog; + BYTE const* const base = ldmState->window.base; + BYTE const* const istart = ip; + ldmRollingHashState_t hashState; + size_t* const splits = ldmState->splitIndices; + unsigned numSplits; + + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); + + ZSTD_ldm_gear_init(&hashState, params); + while (ip < iend) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + if (ip + splits[n] >= istart + minMatchLength) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + ldmEntry_t entry; + + entry.offset = (U32)(split - base); + entry.checksum = (U32)(xxhash >> 32); + ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); + } + } + + ip += hashed; + } +} + + +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +{ + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + curr - MIN(512, curr - ms->nextToUpdate - 1024); + } +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U32 const entsPerBucket = 1U << params->bucketSizeLog; + U32 const hBits = params->hashLog - params->bucketSizeLog; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - HASH_READ_SIZE; + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash state */ + ldmRollingHashState_t hashState; + /* Arrays for staged-processing */ + size_t* const splits = ldmState->splitIndices; + ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; + unsigned numSplits; + + if (srcSize < minMatchLength) + return iend - anchor; + + /* Initialize the rolling hash state with the first minMatchLength bytes */ + ZSTD_ldm_gear_init(&hashState, params); + ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength); + ip += minMatchLength; + + while (ip < ilimit) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, + splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + + candidates[n].split = split; + candidates[n].hash = hash; + candidates[n].checksum = (U32)(xxhash >> 32); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); + PREFETCH_L1(candidates[n].bucket); + } + + for (n = 0; n < numSplits; n++) { + size_t forwardMatchLength = 0, backwardMatchLength = 0, + bestMatchLength = 0, mLength; + U32 offset; + BYTE const* const split = candidates[n].split; + U32 const checksum = candidates[n].checksum; + U32 const hash = candidates[n].hash; + ldmEntry_t* const bucket = candidates[n].bucket; + ldmEntry_t const* cur; + ldmEntry_t const* bestEntry = NULL; + ldmEntry_t newEntry; + + newEntry.offset = (U32)(split - base); + newEntry.checksum = checksum; + + /* If a split point would generate a sequence overlapping with + * the previous one, we merely register it in the hash table and + * move on */ + if (split < anchor) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + for (cur = bucket; cur < bucket + entsPerBucket; cur++) { + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + curForwardMatchLength = + ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( + split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(split, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = + ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); + } + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + + /* No match found -- insert an entry into the hash table + * and process the next candidate match */ + if (bestEntry == NULL) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + /* Match found */ + offset = (U32)(split - base) - bestEntry->offset; + mLength = forwardMatchLength + backwardMatchLength; + { + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; + + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(split - backwardMatchLength - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } + + /* Insert the current entry into the hash table --- it must be + * done after the previous block to avoid clobbering bestEntry */ + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + + anchor = split + forwardMatchLength; + + /* If we find a match that ends after the data that we've hashed + * then we have a repeating, overlapping, pattern. E.g. all zeros. + * If one repetition of the pattern matches our `stopMask` then all + * repetitions will. We don't need to insert them all into out table, + * only the first one. So skip over overlapping matches. + * This is a major speed boost (20x) for compressing a single byte + * repeated, when that byte ends up in the table. + */ + if (anchor > ip + hashed) { + ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength); + /* Continue the outer loop at anchor (ip + hashed == anchor). */ + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + return iend - anchor; +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; + } +} + +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; + + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const *const chunkEnd = + (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; + + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow( + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; + } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal( + ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} + +void +ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +/** + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, + U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} + +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; + + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } + + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the literals */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, + (U32)(iend - ip), minMatch); + /* End signal */ + if (sequence.offset == 0) + break; + + assert(ip + sequence.litLength + sequence.matchLength <= iend); + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); + { + int i; + size_t const newLitLength = + blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i-1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, + OFFSET_TO_OFFBASE(sequence.offset), + sequence.matchLength); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} diff --git a/tools/zstd/lib/compress/zstd_ldm.h b/tools/zstd/lib/compress/zstd_ldm.h new file mode 100644 index 00000000..f147021d --- /dev/null +++ b/tools/zstd/lib/compress/zstd_ldm.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" /* ldmParams_t, U32 */ +#include "../zstd.h" /* ZSTD_CCtx, size_t */ + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + +/** + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); + +/** + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize); + +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); + +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/** ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/tools/zstd/lib/compress/zstd_ldm_geartab.h b/tools/zstd/lib/compress/zstd_ldm_geartab.h new file mode 100644 index 00000000..ef34bc5c --- /dev/null +++ b/tools/zstd/lib/compress/zstd_ldm_geartab.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_GEARTAB_H +#define ZSTD_LDM_GEARTAB_H + +#include "../common/compiler.h" /* UNUSED_ATTR */ +#include "../common/mem.h" /* U64 */ + +static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = { + 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, + 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, + 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, + 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, + 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, + 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, + 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, + 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, + 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, + 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, + 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, + 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, + 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, + 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, + 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, + 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, + 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, + 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, + 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, + 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, + 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, + 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, + 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, + 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, + 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, + 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, + 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, + 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, + 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, + 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, + 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, + 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, + 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, + 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, + 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, + 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, + 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, + 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, + 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, + 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, + 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, + 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, + 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, + 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, + 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, + 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, + 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, + 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, + 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, + 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, + 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, + 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, + 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, + 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, + 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, + 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, + 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, + 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, + 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, + 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, + 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, + 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, + 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, + 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, + 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, + 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, + 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, + 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, + 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, + 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, + 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, + 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, + 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, + 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, + 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, + 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, + 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, + 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, + 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, + 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, + 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, + 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, + 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, + 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, + 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, + 0x2b4da14f2613d8f4 +}; + +#endif /* ZSTD_LDM_GEARTAB_H */ diff --git a/tools/zstd/lib/compress/zstd_opt.c b/tools/zstd/lib/compress/zstd_opt.c new file mode 100644 index 00000000..e63073e5 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_opt.c @@ -0,0 +1,1576 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "hist.h" +#include "zstd_opt.h" + +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) + +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_MAX_PRICE (1<<30) + +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ + + +/*-************************************* +* Price functions for optimal parser +***************************************/ + +#if 0 /* approximation at bit level (for tests) */ +# define BITCOST_ACCURACY 0 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif + +/* ZSTD_bitWeight() : + * provide estimated "cost" of a stat in full bits only */ +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); +} + +/* ZSTD_fracWeight() : + * provide fractional-bit "cost" of a stat, + * using linear interpolation approximation */ +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + /* Fweight was meant for "Fractional weight" + * but it's effectively a value between 1 and 2 + * using fixed point arithmetic */ + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} + +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(int price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} +#endif + +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_ps_disable; +} + +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); +} + + +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; n0); + unsigned const newStat = base + (table[s] >> shift); + sum += newStat; + table[s] = newStat; + } + return sum; +} + +/* ZSTD_scaleStats() : + * reduce all elt frequencies in table if sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); +} + +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) +{ + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; + + if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ + + /* heuristic: use pre-defined stats for too small inputs */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { + DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); + optPtr->priceType = zop_predef; + } + + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + + /* huffman stats covering the full value set : table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; + + if (compressedLiterals) { + /* generate literals statistics from huffman table */ + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + for (lit=0; lit<=MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } } + + { unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll=0; ll<=MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } } + + { unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml=0; ml<=MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } } + + { unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of=0; of<=MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } } + + } else { /* first block, no dictionary */ + + assert(optPtr->litFreq != NULL); + if (compressedLiterals) { + /* base initial cost of literals on direct frequency within src */ + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); + } + + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); + optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + } + + { unsigned ml; + for (ml=0; ml<=MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML+1; + + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); + optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + } + + } + + } else { /* new block : scale down accumulated statistics */ + + if (compressedLiterals) + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); + } + + ZSTD_setBasePrices(optPtr, optLevel); +} + +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, + const optState_t* const optPtr, + int optLevel) +{ + DEBUGLOG(8, "ZSTD_rawLiteralsCost (%u literals)", litLength); + if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + + if (optPtr->priceType == zop_predef) + return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ + + /* dynamic statistics */ + { U32 price = optPtr->litSumBasePrice * litLength; + U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; + U32 u; + assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); + for (u=0; u < litLength; u++) { + U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); + if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; + price -= litPrice; + } + return price; + } +} + +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); + + /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. + * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. + * In such a case, the block would be all literals. + */ + if (litLength == ZSTD_BLOCKSIZE_MAX) + return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); + + /* dynamic statistics */ + { U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } +} + +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence. + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() + * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_getMatchPrice(U32 const offBase, + U32 const matchLength, + const optState_t* const optPtr, + int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(offBase); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); + + if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ + return WEIGHT(mlBase, optLevel) + + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ + + /* dynamic statistics */ + price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel<2) /*static*/ && offCode >= 20) + price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + + /* match Length */ + { U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } + + price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ + + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLength <= iend */ +static void ZSTD_updateStats(optState_t* const optPtr, + U32 litLength, const BYTE* literals, + U32 offBase, U32 matchLength) +{ + /* literals */ + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + } + + /* literal Length */ + { U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* offset code : follows storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(offBase); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } + + /* match Length */ + { U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } +} + + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) +{ + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + *nextToUpdate3 = target; + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. + * @param ip assumed <= iend-8 . + * @param target The target of ZSTD_updateTree_internal() - we are filling to this position + * @return : nb of positions added */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_insertBt1( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + U32 const target, + U32 const mls, const int extDict) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + /* windowLow is based on target because + * we only need positions that will be in the window at the end of the tree update. + */ + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog); + U32 matchEndIdx = curr+8+1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + + assert(curr <= target); + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = curr; /* Update Hash Table */ + + assert(windowLow > 0); + for (; nbCompares && (matchIndex >= windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + + if (!extDict || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); + } +} + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(7, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); + + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); + ms->nextToUpdate = target; +} + +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 +ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + const U32 mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls==3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; + + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = + dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; + + size_t bestLength = lengthToBeat-1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); + + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = curr - repOffset; + U32 repLen = 0; + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { + repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= curr */ + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; + assert(curr >= windowLow); + if ( dictMode == ZSTD_extDict + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", + repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ + matches[mnum].len = (U32)repLen; + mnum++; + if ( (repLen > sufficient_len) + | (ip+repLen == iLimit) ) { /* best possible */ + return mnum; + } } } } + + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); + if ((matchIndex3 >= matchLow) + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } + + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(curr > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = curr+1; /* skip insertion */ + return 1; + } } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } /* if (mls == 3) */ + + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(curr > matchIndex); + + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); + } else { + match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] read */ + } + + if (matchLength > bestLength) { + DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } } + + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } } + + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } } } /* if (dictMode == ZSTD_dictMatchState) */ + + assert(matchEndIdx > curr+8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} + +typedef U32 (*ZSTD_getAllMatchesFn)( + ZSTD_match_t*, + ZSTD_matchState_t*, + U32*, + const BYTE*, + const BYTE*, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat); + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +U32 ZSTD_btGetAllMatches_internal( + ZSTD_match_t* matches, + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, + const BYTE* const iHighLimit, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat, + const ZSTD_dictMode_e dictMode, + const U32 mls) +{ + assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); + DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); + return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); +} + +#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls + +#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ + static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ + ZSTD_match_t* matches, \ + ZSTD_matchState_t* ms, \ + U32* nextToUpdate3, \ + const BYTE* ip, \ + const BYTE* const iHighLimit, \ + const U32 rep[ZSTD_REP_NUM], \ + U32 const ll0, \ + U32 const lengthToBeat) \ + { \ + return ZSTD_btGetAllMatches_internal( \ + matches, ms, nextToUpdate3, ip, iHighLimit, \ + rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ + } + +#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) + +GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) + +#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ + { \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ + } + +static ZSTD_getAllMatchesFn +ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode) +{ + ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { + ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) + }; + U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); + assert((U32)dictMode < 3); + assert(mls - 3 < 4); + return getAllMatchesFns[(int)dictMode][mls - 3]; +} + +/************************* +* LDM helper functions * +*************************/ + +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + rawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in @rawSeqStore by @nbBytes, + * which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) +{ + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void +ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) +{ + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; + + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; + } + /* Calculate appropriate bytes left in matchLength and litLength + * after adjusting based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; + + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); + } +} + +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, + * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', + * into 'matches'. Maintains the correct ordering of 'matches'. + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + const ZSTD_optLdm_t* optLdm, U32 currPosInBlock) +{ + U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ + U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < MINMATCH) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", + candidateOffBase, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffBase; + (*nbMatches)++; + } +} + +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void +ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, + ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes) +{ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + */ + U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + } + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); + } + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); +} + + +/*-******************************* +* Optimal parser +*********************************/ + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + +#define LIT_PRICE(_p) (int)ZSTD_rawLiteralsCost(_p, 1, optStatePtr, optLevel) +#define LL_PRICE(_l) (int)ZSTD_litLengthPrice(_l, optStatePtr, optLevel) +#define LL_INCPRICE(_l) (LL_PRICE(_l) - LL_PRICE(_l-1)) + +FORCE_INLINE_TEMPLATE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t +ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) +{ + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + + ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); + + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; + + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastStretch; + ZSTD_optLdm_t optLdm; + + ZSTD_memset(&lastStretch, 0, sizeof(ZSTD_optimal_t)); + + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); + assert(optLevel <= 2); + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; + + /* find first match */ + { U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend-ip)); + if (!nbMatches) { + DEBUGLOG(8, "no match found at cPos %u", (unsigned)(ip-istart)); + ip++; + continue; + } + + /* Match found: let's store this solution, and eventually find more candidates. + * During this forward pass, @opt is used to store stretches, + * defined as "a match followed by N literals". + * Note how this is different from a Sequence, which is "N literals followed by a match". + * Storing stretches allows us to store different match predecessors + * for each literal position part of a literals run. */ + + /* initialize opt[0] */ + opt[0].mlen = 0; /* there are only literals so far */ + opt[0].litlen = litlen; + /* No need to include the actual price of the literals before the first match + * because it is static for the duration of the forward pass, and is included + * in every subsequent price. But, we include the literal length because + * the cost variation of litlen depends on the value of litlen. + */ + opt[0].price = LL_PRICE(litlen); + ZSTD_STATIC_ASSERT(sizeof(opt[0].rep[0]) == sizeof(rep[0])); + ZSTD_memcpy(&opt[0].rep, rep, sizeof(opt[0].rep)); + + /* large match -> immediate encoding */ + { U32 const maxML = matches[nbMatches-1].len; + U32 const maxOffBase = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); + + if (maxML > sufficient_len) { + lastStretch.litlen = 0; + lastStretch.mlen = maxML; + lastStretch.off = maxOffBase; + DEBUGLOG(6, "large match (%u>%u) => immediate encoding", + maxML, sufficient_len); + cur = 0; + last_pos = maxML; + goto _shortestPath; + } } + + /* set prices for first matches starting position == 0 */ + assert(opt[0].price >= 0); + { U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; + opt[pos].mlen = 0; + opt[pos].litlen = litlen + pos; + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offBase = matches[matchNb].off; + U32 const end = matches[matchNb].len; + for ( ; pos <= end ; pos++ ) { + int const matchPrice = (int)ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); + int const sequencePrice = opt[0].price + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost(sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offBase; + opt[pos].litlen = 0; /* end of match */ + opt[pos].price = sequencePrice + LL_PRICE(0); + } + } + last_pos = pos-1; + opt[pos].price = ZSTD_MAX_PRICE; + } + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur <= ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur); + + /* Fix current position with one literal if cheaper */ + { U32 const litlen = opt[cur-1].litlen + 1; + int const price = opt[cur-1].price + + LIT_PRICE(ip+cur-1) + + LL_INCPRICE(litlen); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + ZSTD_optimal_t const prevMatch = opt[cur]; + DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); + opt[cur] = opt[cur-1]; + opt[cur].litlen = litlen; + opt[cur].price = price; + if ( (optLevel >= 1) /* additional check only for higher modes */ + && (prevMatch.litlen == 0) /* replace a match */ + && (LL_INCPRICE(1) < 0) /* ll1 is cheaper than ll0 */ + && LIKELY(ip + cur < iend) + ) { + /* check next position, in case it would be cheaper */ + int with1literal = prevMatch.price + LIT_PRICE(ip+cur) + LL_INCPRICE(1); + int withMoreLiterals = price + LIT_PRICE(ip+cur) + LL_INCPRICE(litlen+1); + DEBUGLOG(7, "then at next rPos %u : match+1lit %.2f vs %ulits %.2f", + cur+1, ZSTD_fCost(with1literal), litlen+1, ZSTD_fCost(withMoreLiterals)); + if ( (with1literal < withMoreLiterals) + && (with1literal < opt[cur+1].price) ) { + /* update offset history - before it disappears */ + U32 const prev = cur - prevMatch.mlen; + repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, prevMatch.off, opt[prev].litlen==0); + assert(cur >= prevMatch.mlen); + DEBUGLOG(7, "==> match+1lit is cheaper (%.2f < %.2f) (hist:%u,%u,%u) !", + ZSTD_fCost(with1literal), ZSTD_fCost(withMoreLiterals), + newReps.rep[0], newReps.rep[1], newReps.rep[2] ); + opt[cur+1] = prevMatch; /* mlen & offbase */ + ZSTD_memcpy(opt[cur+1].rep, &newReps, sizeof(repcodes_t)); + opt[cur+1].litlen = 1; + opt[cur+1].price = with1literal; + if (last_pos < cur+1) last_pos = cur+1; + } + } + } else { + DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price)); + } + } + + /* Offset history is not updated during match comparison. + * Do it here, now that the match is selected and confirmed. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].litlen == 0) { + /* just finished a match => alter offset history */ + U32 const prev = cur - opt[cur].mlen; + repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[prev].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) continue; + + if (cur == last_pos) break; + + if ( (optLevel==0) /*static_test*/ + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "skip current position : next rPos(%u) price is cheaper", cur+1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + assert(opt[cur].price >= 0); + { U32 const ll0 = (opt[cur].litlen == 0); + int const previousPrice = opt[cur].price; + int const basePrice = previousPrice + LL_PRICE(0); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); + U32 matchNb; + + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); + + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } + + { U32 const longestML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of longest ML=%u", + inr-istart, cur, nbMatches, longestML); + + if ( (longestML > sufficient_len) + || (cur + longestML >= ZSTD_OPT_NUM) + || (ip + cur + longestML >= iend) ) { + lastStretch.mlen = longestML; + lastStretch.off = matches[nbMatches-1].off; + lastStretch.litlen = 0; + last_pos = cur + longestML; + goto _shortestPath; + } } + + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; + U32 mlen; + + DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, opt[cur].litlen); + + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { + /* fill empty positions, for future comparisons */ + last_pos++; + opt[last_pos].price = ZSTD_MAX_PRICE; + opt[last_pos].litlen = !0; /* just needs to be != 0, to mean "not an end of match" */ + } + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = 0; + opt[pos].price = price; + } else { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } } } + opt[last_pos+1].price = ZSTD_MAX_PRICE; + } /* for (cur = 1; cur <= last_pos; cur++) */ + + lastStretch = opt[last_pos]; + assert(cur >= lastStretch.mlen); + cur = last_pos - lastStretch.mlen; + +_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + assert(last_pos >= lastStretch.mlen); + assert(cur == last_pos - lastStretch.mlen); + + if (lastStretch.mlen==0) { + /* no solution : all matches have been converted into literals */ + assert(lastStretch.litlen == (ip - anchor) + last_pos); + ip += last_pos; + continue; + } + assert(lastStretch.off > 0); + + /* Update offset history */ + if (lastStretch.litlen == 0) { + /* finishing on a match : update offset history */ + repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastStretch.off, opt[cur].litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(repcodes_t)); + } else { + ZSTD_memcpy(rep, lastStretch.rep, sizeof(repcodes_t)); + assert(cur >= lastStretch.litlen); + cur -= lastStretch.litlen; + } + + /* Let's write the shortest path solution. + * It is stored in @opt in reverse order, + * starting from @storeEnd (==cur+2), + * effectively partially @opt overwriting. + * Content is changed too: + * - So far, @opt stored stretches, aka a match followed by literals + * - Now, it will store sequences, aka literals followed by a match + */ + { U32 const storeEnd = cur + 2; + U32 storeStart = storeEnd; + U32 stretchPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", + last_pos, cur); (void)last_pos; + assert(storeEnd < ZSTD_OPT_SIZE); + DEBUGLOG(6, "last stretch copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastStretch.litlen, lastStretch.mlen, lastStretch.off); + if (lastStretch.litlen > 0) { + /* last "sequence" is unfinished: just a bunch of literals */ + opt[storeEnd].litlen = lastStretch.litlen; + opt[storeEnd].mlen = 0; + storeStart = storeEnd-1; + opt[storeStart] = lastStretch; + } { + opt[storeEnd] = lastStretch; /* note: litlen will be fixed */ + storeStart = storeEnd; + } + while (1) { + ZSTD_optimal_t nextStretch = opt[stretchPos]; + opt[storeStart].litlen = nextStretch.litlen; + DEBUGLOG(6, "selected sequence (llen=%u,mlen=%u,ofc=%u)", + opt[storeStart].litlen, opt[storeStart].mlen, opt[storeStart].off); + if (nextStretch.mlen == 0) { + /* reaching beginning of segment */ + break; + } + storeStart--; + opt[storeStart] = nextStretch; /* note: litlen will be fixed */ + assert(nextStretch.litlen + nextStretch.mlen <= stretchPos); + stretchPos -= nextStretch.litlen + nextStretch.mlen; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore"); + { U32 storePos; + for (storePos=storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offBase = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", + anchor - istart, (unsigned)llen, (unsigned)mlen); + + if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); + anchor += advance; + ip = anchor; + } } + DEBUGLOG(7, "new offset history : %u, %u, %u", rep[0], rep[1], rep[2]); + + /* update all costs */ + ZSTD_setBasePrices(optStatePtr, optLevel); + } + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} +#endif /* build exclusions */ + +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +static size_t ZSTD_compressBlock_opt0( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +static size_t ZSTD_compressBlock_opt2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} +#endif + + + + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error out, its narrow contract must be respected. + */ +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_initStats_ultra(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history, only keep entropy stats */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + +} + +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 const curr = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-passes strategy: + * this strategy makes a first pass over first block to collect statistics + * in order to seed next round's statistics with it. + * After 1st pass, function forgets history, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + * the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ( (ms->opt.litLengthSum==0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); + } + + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} +#endif + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} +#endif + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/tools/zstd/lib/compress/zstd_opt.h b/tools/zstd/lib/compress/zstd_opt.h new file mode 100644 index 00000000..d4e71131 --- /dev/null +++ b/tools/zstd/lib/compress/zstd_opt.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +#if !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ + || !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); +#endif + +#ifndef ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTOPT ZSTD_compressBlock_btopt +#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE ZSTD_compressBlock_btopt_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT ZSTD_compressBlock_btopt_extDict +#else +#define ZSTD_COMPRESSBLOCK_BTOPT NULL +#define ZSTD_COMPRESSBLOCK_BTOPT_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTOPT_EXTDICT NULL +#endif + +#ifndef ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#define ZSTD_COMPRESSBLOCK_BTULTRA ZSTD_compressBlock_btultra +#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE ZSTD_compressBlock_btultra_dictMatchState +#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT ZSTD_compressBlock_btultra_extDict +#define ZSTD_COMPRESSBLOCK_BTULTRA2 ZSTD_compressBlock_btultra2 +#else +#define ZSTD_COMPRESSBLOCK_BTULTRA NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA_DICTMATCHSTATE NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA_EXTDICT NULL +#define ZSTD_COMPRESSBLOCK_BTULTRA2 NULL +#endif + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_OPT_H */ diff --git a/tools/zstd/lib/compress/zstdmt_compress.c b/tools/zstd/lib/compress/zstdmt_compress.c new file mode 100644 index 00000000..86ccce31 --- /dev/null +++ b/tools/zstd/lib/compress/zstdmt_compress.c @@ -0,0 +1,1882 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/* ====== Dependencies ====== */ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ +#include "../common/mem.h" /* MEM_STATIC */ +#include "../common/pool.h" /* threadpool */ +#include "../common/threading.h" /* mutex */ +#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstd_ldm.h" +#include "zstdmt_compress.h" + +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 + +/* ====== Debug ====== */ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ + && !defined(_MSC_VER) \ + && !defined(__MINGW32__) + +# include +# include +# include + +# define DEBUG_PRINTHEX(l,p,n) \ + do { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ + } while (0) + +static unsigned long long GetCurrentClockTimeMicroseconds(void) +{ + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); + + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); +} } + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) \ + do { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { \ + /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, \ + "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ + } while (0) + +#else + +# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +# define DEBUG_PRINTHEX(l,p,n) do { } while (0) + +#endif + + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t capacity; +} buffer_t; + +static const buffer_t g_nullBuffer = { NULL, 0 }; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + buffer_t* buffers; +} ZSTDMT_bufferPool; + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) +{ + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + if (bufPool->buffers) { + unsigned u; + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->buffers[u].start); + ZSTD_customFree(bufPool->buffers[u].start, bufPool->cMem); + } + ZSTD_customFree(bufPool->buffers, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_customFree(bufPool, bufPool->cMem); +} + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_customMem cMem) +{ + ZSTDMT_bufferPool* const bufPool = + (ZSTDMT_bufferPool*)ZSTD_customCalloc(sizeof(ZSTDMT_bufferPool), cMem); + if (bufPool==NULL) return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_customFree(bufPool, cMem); + return NULL; + } + bufPool->buffers = (buffer_t*)ZSTD_customCalloc(maxNbBuffers * sizeof(buffer_t), cMem); + if (bufPool->buffers==NULL) { + ZSTDMT_freeBufferPool(bufPool); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool); + size_t const arraySize = bufPool->totalBuffers * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->buffers[u].capacity; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + + return poolSize + arraySize + totalBufferSize; +} + +/* ZSTDMT_setBufferSize() : + * all future buffers provided by this buffer pool will have _at least_ this size + * note : it's better for all buffers to have same size, + * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) +{ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); + bufPool->bufferSize = bSize; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); +} + + +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, unsigned maxNbBuffers) +{ + if (srcBufPool==NULL) return NULL; + if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ + return srcBufPool; + /* need a larger buffer pool */ + { ZSTD_customMem const cMem = srcBufPool->cMem; + size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ + ZSTDMT_bufferPool* newBufPool; + ZSTDMT_freeBufferPool(srcBufPool); + newBufPool = ZSTDMT_createBufferPool(maxNbBuffers, cMem); + if (newBufPool==NULL) return newBufPool; + ZSTDMT_setBufferSize(newBufPool, bSize); + return newBufPool; + } +} + +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid + * @return : a buffer, with start pointer and size + * note: allocation may fail, in this case, start==NULL and size==0 */ +static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +{ + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + buffer_t const buf = bufPool->buffers[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.capacity; + bufPool->buffers[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { + /* large enough, but not too much */ + DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", + bufPool->nbBuffers, (U32)buf.capacity); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); + ZSTD_customFree(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); + { buffer_t buffer; + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.capacity = (start==NULL) ? 0 : bSize; + if (start==NULL) { + DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + } else { + DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + } + return buffer; + } +} + +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +{ + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer_t newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + } + return buffer; +} +#endif + +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +{ + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) return; /* compatible with release on NULL */ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->buffers[bufPool->nbBuffers++] = buf; /* stored for later use */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", + (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (note: should not happen) */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); + ZSTD_customFree(buf.start, bufPool->cMem); +} + +/* We need 2 output buffers per worker since each dstBuff must be flushed after it is released. + * The 3 additional buffers are as follows: + * 1 buffer for input loading + * 1 buffer for "next input" when submitting current one + * 1 buffer stuck in queue */ +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) (2*(nbWorkers) + 3) + +/* After a worker releases its rawSeqStore, it is immediately ready for reuse. + * So we only need one seq buffer per worker. */ +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) (nbWorkers) + +/* ===== Seq Pool Wrapper ====== */ + +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); +} + +static rawSeqStore_t bufferToSeq(buffer_t buffer) +{ + rawSeqStore_t seq = kNullRawSeqStore; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; +} + +static buffer_t seqToBuffer(rawSeqStore_t seq) +{ + buffer_t buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} + +static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; + } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} + +#if ZSTD_RESIZE_SEQPOOL +static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif + +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} + +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} + +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(SEQ_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + if (seqPool == NULL) return NULL; + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; +} + +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} + +static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) +{ + return ZSTDMT_expandBufferPool(pool, SEQ_POOL_MAX_NB_BUFFERS(nbWorkers)); +} + + +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ + +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + int totalCCtx; + int availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx** cctxs; +} ZSTDMT_CCtxPool; + +/* note : all CCtx borrowed from the pool must be reverted back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + if (!pool) return; + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + if (pool->cctxs) { + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctxs[cid]); /* free compatible with NULL */ + ZSTD_customFree(pool->cctxs, pool->cMem); + } + ZSTD_customFree(pool, pool->cMem); +} + +/* ZSTDMT_createCCtxPool() : + * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, + ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = + (ZSTDMT_CCtxPool*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtxPool), cMem); + assert(nbWorkers > 0); + if (!cctxPool) return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_customFree(cctxPool, cMem); + return NULL; + } + cctxPool->totalCCtx = nbWorkers; + cctxPool->cctxs = (ZSTD_CCtx**)ZSTD_customCalloc(nbWorkers * sizeof(ZSTD_CCtx*), cMem); + if (!cctxPool->cctxs) { + ZSTDMT_freeCCtxPool(cctxPool); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->cctxs[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctxs[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); + return cctxPool; +} + +static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, + int nbWorkers) +{ + if (srcPool==NULL) return NULL; + if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ + /* need a larger cctx pool */ + { ZSTD_customMem const cMem = srcPool->cMem; + ZSTDMT_freeCCtxPool(srcPool); + return ZSTDMT_createCCtxPool(nbWorkers, cMem); + } +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { unsigned const nbWorkers = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool); + size_t const arraySize = cctxPool->totalCCtx * sizeof(ZSTD_CCtx*); + size_t totalCCtxSize = 0; + unsigned u; + for (u=0; ucctxs[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + assert(nbWorkers > 0); + return poolSize + arraySize + totalCCtxSize; + } +} + +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +{ + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { ZSTD_CCtx* const cctx = cctxPool->cctxs[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} + +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctxs[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbWorkers */ + DEBUGLOG(4, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} + +/* ==== Serial State ==== */ + +typedef struct { + void const* start; + size_t size; +} range_t; + +typedef struct { + /* All variables in the struct are protected by mutex. */ + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + ZSTD_CCtx_params params; + ldmState_t ldmState; + XXH64_state_t xxhState; + unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ +} serialState_t; + +static int +ZSTDMT_serialState_reset(serialState_t* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) +{ + /* Adjust parameters */ + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + } else { + ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } + serialState->nextJobID = 0; + if (params.fParams.checksumFlag) + XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = + params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + unsigned const prevBucketLog = + serialState->params.ldmParams.hashLog - + serialState->params.ldmParams.bucketSizeLog; + size_t const numBuckets = (size_t)1 << bucketLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); + /* Reset the window */ + ZSTD_window_init(&serialState->ldmState.window); + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); + ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets); + + /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; + if (dictSize > 0) { + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ + } + } + + /* Initialize serialState's copy of ldmWindow. */ + serialState->ldmWindow = serialState->ldmState.window; + } + + serialState->params = params; + serialState->params.jobSize = (U32)jobSize; + return 0; +} + +static int ZSTDMT_serialState_init(serialState_t* serialState) +{ + int initError = 0; + ZSTD_memset(serialState, 0, sizeof(*serialState)); + initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); + return initError; +} + +static void ZSTDMT_serialState_free(serialState_t* serialState) +{ + ZSTD_customMem cMem = serialState->params.customMem; + ZSTD_pthread_mutex_destroy(&serialState->mutex); + ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); +} + +static void ZSTDMT_serialState_update(serialState_t* serialState, + ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, + range_t src, unsigned jobID) +{ + /* Wait for our turn */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); + ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + } + /* A future job may error and skip our job */ + if (serialState->nextJobID == jobID) { + /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) { + size_t error; + assert(seqStore.seq != NULL && seqStore.pos == 0 && + seqStore.size == 0 && seqStore.capacity > 0); + assert(src.size <= serialState->params.jobSize); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, &seqStore, + &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + if (serialState->params.fParams.checksumFlag && src.size > 0) + XXH64_update(&serialState->xxhState, src.start, src.size); + } + /* Now it is the next jobs turn */ + serialState->nextJobID++; + ZSTD_pthread_cond_broadcast(&serialState->cond); + ZSTD_pthread_mutex_unlock(&serialState->mutex); + + if (seqStore.size > 0) { + ZSTD_referenceExternalSequences(jobCCtx, seqStore.seq, seqStore.size); + assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); + } +} + +static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, + unsigned jobID, size_t cSize) +{ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + if (serialState->nextJobID <= jobID) { + assert(ZSTD_isError(cSize)); (void)cSize; + DEBUGLOG(5, "Skipping past job %u because of error", jobID); + serialState->nextJobID = jobID + 1; + ZSTD_pthread_cond_broadcast(&serialState->cond); + + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + ZSTD_pthread_mutex_unlock(&serialState->mutex); + +} + + +/* ------------------------------------------ */ +/* ===== Worker thread ===== */ +/* ------------------------------------------ */ + +static const range_t kNullRange = { NULL, 0 }; + +typedef struct { + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ + buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ +} ZSTDMT_jobDescription; + +#define JOB_ERROR(e) \ + do { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ + } while (0) + +/* ZSTDMT_compressionJob() is a POOL_function type */ +static void ZSTDMT_compressionJob(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ + ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + buffer_t dstBuff = job->dstBuff; + size_t lastCBlockSize = 0; + + /* resources */ + if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); + if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); + job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ + } + if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL) + JOB_ERROR(ERROR(memory_allocation)); + + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. + */ + if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = ZSTD_ps_disable; + /* Correct nbWorkers to 0. */ + jobParams.nbWorkers = 0; + + + /* init */ + if (job->cdict) { + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); + assert(job->firstJob); /* only allowed for first job */ + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } else { /* srcStart points at reloaded section */ + U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; + { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); + } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } + { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, + job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ + ZSTD_dtlm_fast, + NULL, /*cdict*/ + &jobParams, pledgedSrcSize); + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } } + + /* Perform serial step as early as possible, but after CCtx initialization */ + ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + + if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ + size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + if (ZSTD_isError(hSize)) JOB_ERROR(hSize); + DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); + ZSTD_invalidateRepCodes(cctx); + } + + /* compress */ + { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; + int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); + const BYTE* ip = (const BYTE*) job->src.start; + BYTE* const ostart = (BYTE*)dstBuff.start; + BYTE* op = ostart; + BYTE* oend = op + dstBuff.capacity; + int chunkNb; + if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ + DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); + assert(job->cSize == 0); + for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { + size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + ip += chunkSize; + op += cSize; assert(op < oend); + /* stats */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + job->cSize += cSize; + job->consumed = chunkSize * chunkNb; + DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", + (U32)cSize, (U32)job->cSize); + ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ + ZSTD_pthread_mutex_unlock(&job->job_mutex); + } + /* last block */ + assert(chunkSize > 0); + assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ + if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { + size_t const lastBlockSize1 = job->src.size & (chunkSize-1); + size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; + size_t const cSize = (job->lastJob) ? + ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + lastCBlockSize = cSize; + } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } + ZSTD_CCtx_trace(cctx, 0); + +_endJob: + ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); + if (job->prefix.size > 0) + DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); + DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); + /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + /* report */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); + job->cSize += lastCBlockSize; + job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ + ZSTD_pthread_cond_signal(&job->job_cond); + ZSTD_pthread_mutex_unlock(&job->job_mutex); +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +typedef struct { + range_t prefix; /* read-only non-owned prefix buffer */ + buffer_t buffer; + size_t filled; +} inBuff_t; + +typedef struct { + BYTE* buffer; /* The round input buffer. All jobs get references + * to pieces of the buffer. ZSTDMT_tryGetInputRange() + * handles handing out job input buffers, and makes + * sure it doesn't overlap with any pieces still in use. + */ + size_t capacity; /* The capacity of buffer. */ + size_t pos; /* The position of the current inBuff in the round + * buffer. Updated past the end if the inBuff once + * the inBuff is sent to the worker thread. + * pos <= capacity. + */ +} roundBuff_t; + +static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; + +#define RSYNC_LENGTH 32 +/* Don't create chunks smaller than the zstd block size. + * This stops us from regressing compression ratio too much, + * and ensures our output fits in ZSTD_compressBound(). + * + * If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then + * ZSTD_COMPRESSBOUND() will need to be updated. + */ +#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX +#define RSYNC_MIN_BLOCK_SIZE (1< one job is already prepared, but pool has shortage of workers. Don't create a new job. */ + inBuff_t inBuff; + roundBuff_t roundBuff; + serialState_t serial; + rsyncState_t rsync; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + unsigned long long consumed; + unsigned long long produced; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + unsigned providedFactory: 1; +}; + +static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) +{ + U32 jobNb; + if (jobTable == NULL) return; + for (jobNb=0; jobNb mtctx->jobIDMask+1) { /* need more job capacity */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + } + return 0; +} + + +/* ZSTDMT_CCtxParam_setNbWorkers(): + * Internal use only */ +static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +{ + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); +} + +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbWorkers + 2; + int initError; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); + + if (nbWorkers < 1) return NULL; + nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); + assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(BUF_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); + initError = ZSTDMT_serialState_init(&mtctx->serial); + mtctx->roundBuff = kNullRoundBuff; + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); +#else + (void)nbWorkers; + (void)cMem; + (void)pool; + return NULL; +#endif +} + + +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + + /* Clear the job description, but keep the mutex/cond */ + ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; + } + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->allJobsCompleted = 1; +} + +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (mtctx->doneJobID < mtctx->nextJobID) { + unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + mtctx->doneJobID++; + } +} + +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx==NULL) return 0; /* compatible with free on NULL */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); + ZSTDMT_serialState_free(&mtctx->serial); + ZSTD_freeCDict(mtctx->cdictLocal); + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_customFree(mtctx, mtctx->cMem); + return 0; +} + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal) + + mtctx->roundBuff.capacity; +} + + +/* ZSTDMT_resize() : + * @return : error code if fails, 0 on success */ +static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +{ + if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, BUF_POOL_MAX_NB_BUFFERS(nbWorkers)); + if (mtctx->bufPool == NULL) return ERROR(memory_allocation); + mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); + if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); + mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); + if (mtctx->seqPool == NULL) return ERROR(memory_allocation); + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + return 0; +} + + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates a selected set of compression parameters, remaining compatible with currently active frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) +{ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", + compressionLevel); + mtctx->params.compressionLevel = compressionLevel; + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; + } +} + +/* ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + * Note : mutex will be acquired during statistics collection inside workers. */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) +{ + ZSTD_frameProgression fps; + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; + fps.consumed = mtctx->consumed; + fps.produced = fps.flushed = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; + fps.nbActiveWorkers = 0; + { unsigned jobNb; + unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); + DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", + mtctx->doneJobID, lastJobNb, mtctx->jobReady); + for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { + unsigned const wJobID = jobNb & mtctx->jobIDMask; + ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + fps.ingested += jobPtr->src.size; + fps.consumed += jobPtr->consumed; + fps.produced += produced; + fps.flushed += flushed; + fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + } + return fps; +} + + +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) +{ + size_t toFlush; + unsigned const jobID = mtctx->doneJobID; + assert(jobID <= mtctx->nextJobID); + if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ + + /* look into oldest non-fully-flushed job */ + { unsigned const wJobID = jobID & mtctx->jobIDMask; + ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); + toFlush = produced - flushed; + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { + assert(jobPtr->consumed < jobPtr->src.size); + } + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + + return toFlush; +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) +{ + unsigned jobLog; + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on cycleLog instead. */ + jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); + } else { + jobLog = MAX(20, params->cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); +} + +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) +{ + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; +} + +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +{ + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); + return ovlog; +} + +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + - overlapRLog; + } + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog==0) ? 0 : (size_t)1 << ovLog; +} + +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ + +size_t ZSTDMT_initCStream_internal( + ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); + + /* params supposed partially fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + /* init */ + if (params.nbWorkers != mtctx->params.nbWorkers) + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); + + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; + + DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); + + if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + mtctx->allJobsCompleted = 1; + } + + mtctx->params = params; + mtctx->frameContentSize = pledgedSrcSize; + if (dict) { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } + + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); + mtctx->targetSectionSize = params.jobSize; + if (mtctx->targetSectionSize == 0) { + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); + } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10); + U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10); + /* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our + * expected job size is at least 4x larger. */ + assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } + if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); + DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); + { + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ + size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; + if (mtctx->roundBuff.capacity < capacity) { + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); + if (mtctx->roundBuff.buffer == NULL) { + mtctx->roundBuff.capacity = 0; + return ERROR(memory_allocation); + } + mtctx->roundBuff.capacity = capacity; + } + } + DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10)); + mtctx->roundBuff.pos = 0; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->inBuff.prefix = kNullRange; + mtctx->doneJobID = 0; + mtctx->nextJobID = 0; + mtctx->frameEnded = 0; + mtctx->allJobsCompleted = 0; + mtctx->consumed = 0; + mtctx->produced = 0; + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) + return ERROR(memory_allocation); + return 0; +} + + +/* ZSTDMT_writeLastEmptyBlock() + * Write a single empty block with an end-of-frame to finish a frame. + * Job must be created from streaming variant. + * This function is always successful if expected conditions are fulfilled. + */ +static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) +{ + assert(job->lastJob == 1); + assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ + assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ + assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ + job->dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (job->dstBuff.start == NULL) { + job->cSize = ERROR(memory_allocation); + return; + } + assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ + job->src = kNullRange; + job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); + assert(!ZSTD_isError(job->cSize)); + assert(job->consumed == 0); +} + +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) +{ + unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; + int const endFrame = (endOp == ZSTD_e_end); + + if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); + assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); + return 0; + } + + if (!mtctx->jobReady) { + BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; + DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", + mtctx->nextJobID, (U32)srcSize, (U32)mtctx->inBuff.prefix.size); + mtctx->jobs[jobID].src.start = src; + mtctx->jobs[jobID].src.size = srcSize; + assert(mtctx->inBuff.filled >= srcSize); + mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; + mtctx->jobs[jobID].consumed = 0; + mtctx->jobs[jobID].cSize = 0; + mtctx->jobs[jobID].params = mtctx->params; + mtctx->jobs[jobID].cdict = mtctx->nextJobID==0 ? mtctx->cdict : NULL; + mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; + mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; + mtctx->jobs[jobID].serial = &mtctx->serial; + mtctx->jobs[jobID].jobID = mtctx->nextJobID; + mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); + mtctx->jobs[jobID].lastJob = endFrame; + mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0); + mtctx->jobs[jobID].dstFlushed = 0; + + /* Update the round buffer pos and clear the input buffer to be reset */ + mtctx->roundBuff.pos += srcSize; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + /* Set the prefix */ + if (!endFrame) { + size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); + mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; + mtctx->inBuff.prefix.size = newPrefixSize; + } else { /* endFrame==1 => no need for another input buffer */ + mtctx->inBuff.prefix = kNullRange; + mtctx->frameEnded = endFrame; + if (mtctx->nextJobID == 0) { + /* single job exception : checksum is already calculated directly within worker thread */ + mtctx->params.fParams.checksumFlag = 0; + } } + + if ( (srcSize == 0) + && (mtctx->nextJobID>0)/*single job must also write frame header*/ ) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); + assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ + ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); + mtctx->nextJobID++; + return 0; + } + } + + DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", + mtctx->nextJobID, + (U32)mtctx->jobs[jobID].src.size, + mtctx->jobs[jobID].lastJob, + mtctx->nextJobID, + jobID); + if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { + mtctx->nextJobID++; + mtctx->jobReady = 0; + } else { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); + mtctx->jobReady = 1; + } + return 0; +} + + +/*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. + * `output` : `pos` will be updated with amount of data flushed . + * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ +static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +{ + unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; + DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", + blockToFlush, mtctx->doneJobID, mtctx->nextJobID); + assert(output->size >= output->pos); + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + if ( blockToFlush + && (mtctx->doneJobID < mtctx->nextJobID) ) { + assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); + while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ + if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { + DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].consumed, (U32)mtctx->jobs[wJobID].src.size); + break; + } + DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ + } } + + /* try to flush something */ + { size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ + size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + if (ZSTD_isError(cSize)) { + DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", + mtctx->doneJobID, ZSTD_getErrorName(cSize)); + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + return cSize; + } + /* add frame checksum if necessary (can only happen once) */ + assert(srcConsumed <= srcSize); + if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ + && mtctx->jobs[wJobID].frameChecksumNeeded ) { + U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); + DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); + cSize += 4; + mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ + mtctx->jobs[wJobID].frameChecksumNeeded = 0; + } + + if (cSize > 0) { /* compression is ongoing or completed */ + size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); + DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", + (U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize); + assert(mtctx->doneJobID < mtctx->nextJobID); + assert(cSize >= mtctx->jobs[wJobID].dstFlushed); + assert(mtctx->jobs[wJobID].dstBuff.start != NULL); + if (toFlush > 0) { + ZSTD_memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + } + output->pos += toFlush; + mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ + + if ( (srcConsumed == srcSize) /* job is completed */ + && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ + DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released"); + mtctx->jobs[wJobID].dstBuff = g_nullBuffer; + mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ + mtctx->consumed += srcSize; + mtctx->produced += cSize; + mtctx->doneJobID++; + } } + + /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ + if (cSize > mtctx->jobs[wJobID].dstFlushed) return (cSize - mtctx->jobs[wJobID].dstFlushed); + if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ + } + if (mtctx->doneJobID < mtctx->nextJobID) return 1; /* some more jobs ongoing */ + if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ + if (mtctx->inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ + mtctx->allJobsCompleted = mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ + if (end == ZSTD_e_end) return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal buffers fully flushed ? */ + return 0; /* internal buffers fully flushed */ +} + +/** + * Returns the range of data used by the earliest job that is not yet complete. + * If the data of the first job is broken up into two segments, we cover both + * sections. + */ +static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +{ + unsigned const firstJobID = mtctx->doneJobID; + unsigned const lastJobID = mtctx->nextJobID; + unsigned jobID; + + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { + unsigned const wJobID = jobID & mtctx->jobIDMask; + size_t consumed; + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + consumed = mtctx->jobs[wJobID].consumed; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + + if (consumed < mtctx->jobs[wJobID].src.size) { + range_t range = mtctx->jobs[wJobID].prefix; + if (range.size == 0) { + /* Empty prefix */ + range = mtctx->jobs[wJobID].src; + } + /* Job source in multiple segments not supported yet */ + assert(range.start <= mtctx->jobs[wJobID].src.start); + return range; + } + } + return kNullRange; +} + +/** + * Returns non-zero iff buffer and range overlap. + */ +static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +{ + BYTE const* const bufferStart = (BYTE const*)buffer.start; + BYTE const* const rangeStart = (BYTE const*)range.start; + + if (rangeStart == NULL || bufferStart == NULL) + return 0; + + { + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeEnd = rangeStart + range.size; + + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; + + return bufferStart < rangeEnd && rangeStart < bufferEnd; + } +} + +static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +{ + range_t extDict; + range_t prefix; + + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; + + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", + (size_t)extDict.start, + (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", + (size_t)prefix.start, + (size_t)prefix.start + prefix.size); + + return ZSTDMT_isOverlapped(buffer, extDict) + || ZSTDMT_isOverlapped(buffer, prefix); +} + +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +{ + if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); + DEBUGLOG(5, "source [0x%zx, 0x%zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(5, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} + +/** + * Attempts to set the inBuff to the next section to fill. + * If any part of the new section is still in use we give up. + * Returns non-zero if the buffer is filled. + */ +static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +{ + range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); + size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; + size_t const target = mtctx->targetSectionSize; + buffer_t buffer; + + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); + assert(mtctx->inBuff.buffer.start == NULL); + assert(mtctx->roundBuff.capacity >= target); + + if (spaceLeft < target) { + /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. + * Simply copy the prefix to the beginning in that case. + */ + BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; + size_t const prefixSize = mtctx->inBuff.prefix.size; + + buffer.start = start; + buffer.capacity = prefixSize; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + ZSTDMT_waitForLdmComplete(mtctx, buffer); + ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); + mtctx->inBuff.prefix.start = start; + mtctx->roundBuff.pos = prefixSize; + } + buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; + buffer.capacity = target; + + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + + ZSTDMT_waitForLdmComplete(mtctx, buffer); + + DEBUGLOG(5, "Using prefix range [%zx, %zx)", + (size_t)mtctx->inBuff.prefix.start, + (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); + DEBUGLOG(5, "Using source range [%zx, %zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + + + mtctx->inBuff.buffer = buffer; + mtctx->inBuff.filled = 0; + assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); + return 1; +} + +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} syncPoint_t; + +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static syncPoint_t +findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) +{ + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; + + syncPoint_t syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; + + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE) + /* We don't emit synchronization points if it would produce too small blocks. + * We don't have enough input to find a synchronization point, so don't look. + */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) { + /* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions + * because they can't possibly be a sync point. So we can start + * part way through the input buffer. + */ + pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled; + if (pos >= RSYNC_LENGTH) { + prev = istart + pos - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + assert(mtctx->inBuff.filled >= RSYNC_LENGTH); + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos)); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + } else { + /* We have enough bytes buffered to initialize the hash, + * and have processed enough bytes to find a sync point. + * Start scanning at the beginning of the input. + */ + assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); + assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH); + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + if ((hash & hitMask) == hitMask) { + /* We're already at a sync point so don't load any more until + * we're able to flush this sync point. + * This likely happened because the job table was full so we + * couldn't add our job. + */ + syncPoint.toLoad = 0; + syncPoint.flush = 1; + return syncPoint; + } + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + /* This assert is very expensive, and Debian compiles with asserts enabled. + * So disable it for now. We can get similar coverage by checking it at the + * beginning & end of the loop. + * assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + */ + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + ++pos; /* for assert */ + break; + } + } + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + return syncPoint; +} + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) +{ + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize==0) hintInSize = mtctx->targetSectionSize; + return hintInSize; +} + +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + unsigned forwardInputProgress = 0; + DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", + (U32)endOp, (U32)(input->size - input->pos)); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); + } + + /* fill input buffer */ + if ( (!mtctx->jobReady) + && (input->size > input->pos) ) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ + if (!ZSTDMT_tryGetInputRange(mtctx)) { + /* It is only possible for this operation to fail if there are + * still compression jobs ongoing. + */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); + assert(mtctx->doneJobID != mtctx->nextJobID); + } else + DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); + } + if (mtctx->inBuff.buffer.start != NULL) { + syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } + assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); + DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", + (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); + ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad>0; + } + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. + * - We filled the input buffer: flush this job but don't end the frame. + * - We hit a synchronization point: flush this job but don't end the frame. + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; + } + + if ( (mtctx->jobReady) + || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ + || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ + || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ + size_t const jobSize = mtctx->inBuff.filled; + assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); + } + + /* check for potential compressed data ready to be flushed */ + { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ + if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); + return remainingToFlush; + } +} diff --git a/tools/zstd/lib/compress/zstdmt_compress.h b/tools/zstd/lib/compress/zstdmt_compress.h new file mode 100644 index 00000000..ed4dc0e9 --- /dev/null +++ b/tools/zstd/lib/compress/zstdmt_compress.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + #ifndef ZSTDMT_COMPRESS_H + #define ZSTDMT_COMPRESS_H + + #if defined (__cplusplus) + extern "C" { + #endif + + +/* Note : This is an internal API. + * These APIs used to be exposed with ZSTDLIB_API, + * because it used to be the only way to invoke MT compression. + * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. + * + * This API requires ZSTD_MULTITHREAD to be defined during compilation, + * otherwise ZSTDMT_createCCtx*() will fail. + */ + +/* === Dependencies === */ +#include "../common/zstd_deps.h" /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ + + +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */ +# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) +#endif +#ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */ +# define ZSTDMT_JOBSIZE_MIN (512 KB) +#endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + + +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ + +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, + ZSTD_customMem cMem, + ZSTD_threadPool *pool); +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + +/* === Streaming functions === */ + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * mtctx can be freshly constructed or reused from a prior compression. + * If mtctx is reused, memory allocations from the prior compression may not be freed, + * even if they are not needed for the current compression. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code + * note : needs to be init using any ZSTD_initCStream*() variant */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + /*! ZSTDMT_toFlushNow() + * Tell how many bytes are ready to be flushed immediately. + * Probe the oldest active job (not yet entirely flushed) and check its output buffer. + * If return 0, it means there is no active job, + * or, it means oldest job is still active, but everything produced has been flushed so far, + * therefore flushing is limited by speed of oldest job. */ +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates only a selected set of compression parameters, to remain compatible with current frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); + +/*! ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDMT_COMPRESS_H */ diff --git a/tools/zstd/lib/decompress/huf_decompress.c b/tools/zstd/lib/decompress/huf_decompress.c new file mode 100644 index 00000000..f85dd0be --- /dev/null +++ b/tools/zstd/lib/decompress/huf_decompress.c @@ -0,0 +1,1944 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/zstd_internal.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 +#else +# define HUF_ENABLE_FAST_DECODE 1 +#endif + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilowest; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const istart = (const BYTE*)src; + + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + */ + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. + */ + args->ilowest = istart; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) + +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return (size_t)(pEnd-pStart); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) + +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); + + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); + +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilowest and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } + +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} + +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#endif + } +} diff --git a/tools/zstd/lib/decompress/huf_decompress_amd64.S b/tools/zstd/lib/decompress/huf_decompress_amd64.S new file mode 100644 index 00000000..78da291e --- /dev/null +++ b/tools/zstd/lib/decompress/huf_decompress_amd64.S @@ -0,0 +1,595 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "../common/portability_macros.h" + +#if defined(__ELF__) && defined(__GNUC__) +/* Stack marking + * ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart + */ +.section .note.GNU-stack,"",%progbits + +#if defined(__aarch64__) +/* Mark that this assembly supports BTI & PAC, because it is empty for aarch64. + * See: https://github.com/facebook/zstd/issues/3841 + * See: https://gcc.godbolt.org/z/sqr5T4ffK + * See: https://lore.kernel.org/linux-arm-kernel/20200429211641.9279-8-broonie@kernel.org/ + * See: https://reviews.llvm.org/D62609 + */ +.pushsection .note.gnu.property, "a" +.p2align 3 +.long 4 /* size of the name - "GNU\0" */ +.long 0x10 /* size of descriptor */ +.long 0x5 /* NT_GNU_PROPERTY_TYPE_0 */ +.asciz "GNU" +.long 0xc0000000 /* pr_type - GNU_PROPERTY_AARCH64_FEATURE_1_AND */ +.long 4 /* pr_datasz - 4 bytes */ +.long 3 /* pr_data - GNU_PROPERTY_AARCH64_FEATURE_1_BTI | GNU_PROPERTY_AARCH64_FEATURE_1_PAC */ +.p2align 3 /* pr_padding - bring everything to 8 byte alignment */ +.popsection +#endif + +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +/* Calling convention: + * + * %rdi contains the first argument: HUF_DecompressAsmArgs*. + * %rbp isn't maintained (no frame pointer). + * %rsp contains the stack pointer that grows down. + * No red-zone is assumed, only addresses >= %rsp are used. + * All register contents are preserved. + * + * TODO: Support Windows calling convention. + */ + +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.text + +/* Sets up register mappings for clarity. + * op[], bits[], dtable & ip[0] each get their own register. + * ip[1,2,3] & olimit alias var[]. + * %rax is a scratch register. + */ + +#define op0 rsi +#define op1 rbx +#define op2 rcx +#define op3 rdi + +#define ip0 r8 +#define ip1 r9 +#define ip2 r10 +#define ip3 r11 + +#define bits0 rbp +#define bits1 rdx +#define bits2 r12 +#define bits3 r13 +#define dtable r14 +#define olimit r15 + +/* var[] aliases ip[1,2,3] & olimit + * ip[1,2,3] are saved every iteration. + * olimit is only used in compute_olimit. + */ +#define var0 r15 +#define var1 r9 +#define var2 r10 +#define var3 r11 + +/* 32-bit var registers */ +#define vard0 r15d +#define vard1 r9d +#define vard2 r10d +#define vard3 r11d + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM(X) \ + X(0); \ + X(1); \ + X(2); \ + X(3) + +/* Calls X(N, idx) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \ + X(0, idx); \ + X(1, idx); \ + X(2, idx); \ + X(3, idx) + +/* Define both _HUF_* & HUF_* symbols because MacOS + * C symbols are prefixed with '_' & Linux symbols aren't. + */ +_HUF_decompress4X1_usingDTable_internal_fast_asm_loop: +HUF_decompress4X1_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + /* Read HUF_DecompressAsmArgs* args from %rax */ + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push 104(%rax) /* ilowest */ + push 112(%rax) /* oend */ + push %olimit /* olimit space */ + + subq $24, %rsp + +.L_4X1_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rbx, rdx must be saved + * op3 & ip0 mustn't be clobbered + */ + movq %rbx, 0(%rsp) + movq %rdx, 8(%rsp) + + movq 32(%rsp), %rax /* rax = oend */ + subq %op3, %rax /* rax = oend - op3 */ + + /* r15 = (oend - op3) / 5 */ + movabsq $-3689348814741910323, %rdx + mulq %rdx + movq %rdx, %r15 + shrq $2, %r15 + + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilowest */ + subq %rdx, %rax /* rax = ip0 - ilowest */ + movq %rax, %rbx /* rbx = ip0 - ilowest */ + + /* rdx = (ip0 - ilowest) / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %rbx + shrq %rbx + addq %rbx, %rdx + shrq $2, %rdx + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* r15 = r15 * 5 */ + leaq (%r15, %r15, 4), %r15 + + /* olimit = op3 + r15 */ + addq %op3, %olimit + + movq 8(%rsp), %rdx + movq 0(%rsp), %rbx + + /* If (op3 + 20 > olimit) */ + movq %op3, %rax /* rax = op3 */ + cmpq %rax, %olimit /* op3 == olimit */ + je .L_4X1_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X1_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X1_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X1_exit + +/* Reads top 11 bits from bits[n] + * Loads dt[bits[n]] into var[n] + */ +#define GET_NEXT_DELT(n) \ + movq $53, %var##n; \ + shrxq %var##n, %bits##n, %var##n; \ + movzwl (%dtable,%var##n,2),%vard##n + +/* var[n] must contain the DTable entry computed with GET_NEXT_DELT + * Moves var[n] to %rax + * bits[n] <<= var[n] & 63 + * op[n][idx] = %rax >> 8 + * %ah is a way to access bits [8, 16) of %rax + */ +#define DECODE_FROM_DELT(n, idx) \ + movq %var##n, %rax; \ + shlxq %var##n, %bits##n, %bits##n; \ + movb %ah, idx(%op##n) + +/* Assumes GET_NEXT_DELT has been called. + * Calls DECODE_FROM_DELT then GET_NEXT_DELT + */ +#define DECODE_AND_GET_NEXT(n, idx) \ + DECODE_FROM_DELT(n, idx); \ + GET_NEXT_DELT(n) \ + +/* // ctz & nbBytes is stored in bits[n] + * // nbBits is stored in %rax + * ctz = CTZ[bits[n]] + * nbBits = ctz & 7 + * nbBytes = ctz >> 3 + * op[n] += 5 + * ip[n] -= nbBytes + * // Note: x86-64 is little-endian ==> no bswap + * bits[n] = MEM_readST(ip[n]) | 1 + * bits[n] <<= nbBits + */ +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + andq $7, %rax; \ + shrq $3, %bits##n; \ + leaq 5(%op##n), %op##n; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlx %rax, %bits##n, %bits##n + + /* Store clobbered variables on the stack */ + movq %olimit, 24(%rsp) + movq %ip1, 0(%rsp) + movq %ip2, 8(%rsp) + movq %ip3, 16(%rsp) + + /* Call GET_NEXT_DELT for each stream */ + FOR_EACH_STREAM(GET_NEXT_DELT) + + .p2align 6 + +.L_4X1_loop_body: + /* Decode 5 symbols in each of the 4 streams (20 total) + * Must have called GET_NEXT_DELT for each stream + */ + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4) + + /* Load ip[1,2,3] from stack (var[] aliases them) + * ip[] is needed for RELOAD_BITS + * Each will be stored back to the stack after RELOAD + */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Reload each stream & fetch the next table entry + * to prepare for the next iteration + */ + RELOAD_BITS(0) + GET_NEXT_DELT(0) + + RELOAD_BITS(1) + movq %ip1, 0(%rsp) + GET_NEXT_DELT(1) + + RELOAD_BITS(2) + movq %ip2, 8(%rsp) + GET_NEXT_DELT(2) + + RELOAD_BITS(3) + movq %ip3, 16(%rsp) + GET_NEXT_DELT(3) + + /* If op3 < olimit: continue the loop */ + cmp %op3, 24(%rsp) + ja .L_4X1_loop_body + + /* Reload ip[1,2,3] from stack */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Re-compute olimit */ + jmp .L_4X1_compute_olimit + +#undef GET_NEXT_DELT +#undef DECODE_FROM_DELT +#undef DECODE +#undef RELOAD_BITS +.L_4X1_exit: + addq $24, %rsp + + /* Restore stack (oend & olimit) */ + pop %rax /* olimit */ + pop %rax /* oend */ + pop %rax /* ilowest */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +_HUF_decompress4X2_usingDTable_internal_fast_asm_loop: +HUF_decompress4X2_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push %rax /* olimit */ + push 104(%rax) /* ilowest */ + + movq 112(%rax), %rax + push %rax /* oend3 */ + + movq %op3, %rax + push %rax /* oend2 */ + + movq %op2, %rax + push %rax /* oend1 */ + + movq %op1, %rax + push %rax /* oend0 */ + + /* Scratch space */ + subq $8, %rsp + +.L_4X2_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rdx must be saved + * op[1,2,3,4] & ip0 mustn't be clobbered + */ + movq %rdx, 0(%rsp) + + /* We can consume up to 7 input bytes each iteration. */ + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilowest */ + subq %rdx, %rax /* rax = ip0 - ilowest */ + movq %rax, %r15 /* r15 = ip0 - ilowest */ + + /* rdx = rax / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %r15 + shrq %r15 + addq %r15, %rdx + shrq $2, %rdx + + /* r15 = (ip0 - ilowest) / 7 */ + movq %rdx, %r15 + + /* r15 = min(r15, min(oend0 - op0, oend1 - op1, oend2 - op2, oend3 - op3) / 10) */ + movq 8(%rsp), %rax /* rax = oend0 */ + subq %op0, %rax /* rax = oend0 - op0 */ + movq 16(%rsp), %rdx /* rdx = oend1 */ + subq %op1, %rdx /* rdx = oend1 - op1 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 24(%rsp), %rax /* rax = oend2 */ + subq %op2, %rax /* rax = oend2 - op2 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 32(%rsp), %rax /* rax = oend3 */ + subq %op3, %rax /* rax = oend3 - op3 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movabsq $-3689348814741910323, %rax + mulq %rdx + shrq $3, %rdx /* rdx = rdx / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* olimit = op3 + 5 * r15 */ + movq %r15, %rax + leaq (%op3, %rax, 4), %olimit + addq %rax, %olimit + + movq 0(%rsp), %rdx + + /* If (op3 + 10 > olimit) */ + movq %op3, %rax /* rax = op3 */ + cmpq %rax, %olimit /* op3 == olimit */ + je .L_4X2_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X2_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X2_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X2_exit + +#define DECODE(n, idx) \ + movq %bits##n, %rax; \ + shrq $53, %rax; \ + movzwl 0(%dtable,%rax,4),%r8d; \ + movzbl 2(%dtable,%rax,4),%r15d; \ + movzbl 3(%dtable,%rax,4),%eax; \ + movw %r8w, (%op##n); \ + shlxq %r15, %bits##n, %bits##n; \ + addq %rax, %op##n + +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + shrq $3, %bits##n; \ + andq $7, %rax; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlxq %rax, %bits##n, %bits##n + + + movq %olimit, 48(%rsp) + + .p2align 6 + +.L_4X2_loop_body: + /* We clobber r8, so store it on the stack */ + movq %r8, 0(%rsp) + + /* Decode 5 symbols from each of the 4 streams (20 symbols total). */ + FOR_EACH_STREAM_WITH_INDEX(DECODE, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 4) + + /* Reload r8 */ + movq 0(%rsp), %r8 + + FOR_EACH_STREAM(RELOAD_BITS) + + cmp %op3, 48(%rsp) + ja .L_4X2_loop_body + jmp .L_4X2_compute_olimit + +#undef DECODE +#undef RELOAD_BITS +.L_4X2_exit: + addq $8, %rsp + /* Restore stack (oend & olimit) */ + pop %rax /* oend0 */ + pop %rax /* oend1 */ + pop %rax /* oend2 */ + pop %rax /* oend3 */ + pop %rax /* ilowest */ + pop %rax /* olimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +#endif diff --git a/tools/zstd/lib/decompress/zstd_ddict.c b/tools/zstd/lib/decompress/zstd_ddict.c new file mode 100644 index 00000000..309ec0d0 --- /dev/null +++ b/tools/zstd/lib/decompress/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ddict->dictID; +} diff --git a/tools/zstd/lib/decompress/zstd_ddict.h b/tools/zstd/lib/decompress/zstd_ddict.h new file mode 100644 index 00000000..c4ca8877 --- /dev/null +++ b/tools/zstd/lib/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/tools/zstd/lib/decompress/zstd_decompress.c b/tools/zstd/lib/decompress/zstd_decompress.c new file mode 100644 index 00000000..2f03cf7b --- /dev/null +++ b/tools/zstd/lib/decompress/zstd_decompress.c @@ -0,0 +1,2407 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/error_private.h" +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "../common/mem.h" /* low level memory routines */ +#include "../common/bits.h" /* ZSTD_highbit32 */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; + dctx->isFrameDecompression = 1; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + +/** ZSTD_findDecompressedSize() : + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; + + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; + } + /* skip to next frame */ + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.nbBlocks = nbBlocks; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : (unsigned long long)nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_findFrameCompressedSize() : + * See docs in zstd.h + * Note: compatible with legacy mode */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memmove(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +#endif +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + + /* Loop on each block */ + while (1) { + BYTE* oBlockEnd = oend; + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + + switch(blockProperties.blockType) + { + case bt_compressed: + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); + break; + case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { + XXH64_update(&dctx->xxhState, op, decodedSize); + } + if (decodedSize) /* support dst = NULL,0 */ { + op += decodedSize; + } + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; /* check next frame */ + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return BOUNDED(1, inputSize, dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize, /* flags */ 0); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize, /* flags */ 0); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + DEBUGLOG(4, "ZSTD_resetDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) +{ + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); + ip = istart + cSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { + size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + + op = op ? op + flushedSize : op; + + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} diff --git a/tools/zstd/lib/decompress/zstd_decompress_block.c b/tools/zstd/lib/decompress/zstd_decompress_block.c new file mode 100644 index 00000000..76d7332e --- /dev/null +++ b/tools/zstd/lib/decompress/zstd_decompress_block.c @@ -0,0 +1,2215 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} + +/*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } else { + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); + } + } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what empirically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold)) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) +{ + seq_t seq; + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); + } + } + + return seq; +} + +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} +#endif + +static void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); + + /* Literals are split between internal buffer & output buffer */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + + /* check if reached exact end */ + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + } + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + + /* finish queue */ + seqNb -= seqAdvance; + for ( ; seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return (size_t)(op - ostart); +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + } + + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } +} + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const streaming_operation streaming) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); + + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } + } + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + } + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + } +} + + +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + dctx->isFrameDecompression = 0; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} diff --git a/tools/zstd/lib/decompress/zstd_decompress_block.h b/tools/zstd/lib/decompress/zstd_decompress_block.h new file mode 100644 index 00000000..ab152404 --- /dev/null +++ b/tools/zstd/lib/decompress/zstd_decompress_block.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const streaming_operation streaming); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/tools/zstd/lib/decompress/zstd_decompress_internal.h b/tools/zstd/lib/decompress/zstd_decompress_internal.h new file mode 100644 index 00000000..83a7a011 --- /dev/null +++ b/tools/zstd/lib/decompress/zstd_decompress_internal.h @@ -0,0 +1,240 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int isFrameDecompression; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + int maxBlockSizeParam; + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/tools/zstd/lib/zstd.h b/tools/zstd/lib/zstd.h new file mode 100644 index 00000000..5d1fef8a --- /dev/null +++ b/tools/zstd/lib/zstd.h @@ -0,0 +1,3089 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependencies ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 6 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +/* ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()` or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters (see below). + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximatively targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_useBlockSplitter + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will reuse the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be reused multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder finished and flushed all remaining buffers. + * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers, + * call ZSTD_decompressStream() again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsSize The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). + */ +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * + * When srcSize can be bound by a known and rather "small" value, + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * reuse decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be reused multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be reused multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/tools/zstd/lib/zstd_errors.h b/tools/zstd/lib/zstd_errors.h new file mode 100644 index 00000000..dc75eeeb --- /dev/null +++ b/tools/zstd/lib/zstd_errors.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ From 0cb008525d0bbb0fc9006dd5b2e57c760bea0f5b Mon Sep 17 00:00:00 2001 From: Kai Zhao Date: Tue, 6 Oct 2026 16:48:32 -0400 Subject: [PATCH 2/3] Add SZ3 filter plugin 32024 Build the H5Z-SZ3 filter with SZ3's own CMake from the SZ3 v3.4.0 git subtree in SZ3/SZ3, following the ZFP/ layout: SZ3/CMakeLists.txt is a thin wrapper that adds SZ3 with BUILD_H5Z_FILTER and its bundled Zstd, installs the plugin to lib/plugin through H5Z_SZ3_PLUGIN_INSTALL_DIR, copies it into plugins/ for testing and aliases hdf5sz3 to h5sz3. SZ3 needs no external library, so the filter also builds with H5PL_ALLOW_EXTERNAL_SUPPORT=NO. Includes an example program, h5repack/h5dump tests, the SZ3 and Zstd license notices installed with the plugin, the docs for the subtree, and ENABLE_SZ3 in the workflows. --- .github/workflows/clang-format-check.yml | 2 +- .github/workflows/clang-format-fix.yml | 2 +- .github/workflows/main-fetchcontent-mac.yml | 3 + .../workflows/main-fetchcontent-ubuntu.yml | 3 + .github/workflows/main-fetchcontent-win.yml | 3 + .../workflows/main-package-managers-mac.yml | 1 + .../main-package-managers-ubuntu.yml | 1 + .../workflows/main-package-managers-win.yml | 1 + CMakeLists.txt | 8 + .../copyright-and-BSD-license.txt | 29 + SZ3/Additional_Legal/zstd-LICENSE | 30 + SZ3/CMakeLists.txt | 204 ++++ SZ3/CTestConfig.cmake | 36 + SZ3/README.md | 129 +++ SZ3/UPDATING_SZ3_SUBTREE.md | 193 ++++ SZ3/config/cmake-presets/hidden-presets.json | 514 +++++++++ SZ3/config/cmake/CTestCustom.cmake | 23 + SZ3/config/cmake/CTestScript.cmake | 219 ++++ SZ3/config/cmake/ConfigureChecks.cmake | 346 +++++++ SZ3/config/cmake/H5PLTests.c | 176 ++++ SZ3/config/cmake/H5PL_Examples.cmake.in | 128 +++ SZ3/config/cmake/HDFMacros.cmake | 519 ++++++++++ SZ3/config/cmake/HDFPLoptions.cmake | 21 + SZ3/config/cmake/HDFPluginMacros.cmake | 577 +++++++++++ SZ3/config/cmake/NSIS.InstallOptions.ini.in | 37 + SZ3/config/cmake/NSIS.template.in | 974 ++++++++++++++++++ SZ3/config/cmake/README.txt.cmake.in | 105 ++ SZ3/config/cmake/SignPackageFiles.cmake | 43 + SZ3/config/cmake/binex/CMakeLists.txt | 47 + SZ3/config/cmake/binex/CMakePresets.json | 226 ++++ SZ3/config/cmake/binex/CTestConfig.cmake | 29 + SZ3/config/cmake/binex/Using_CMake.txt | 259 +++++ .../config/cmake-presets/hidden-presets.json | 514 +++++++++ .../binex/config/cmake/CTestCustom.cmake | 22 + .../cmake/binex/config/cmake/HDFMacros.cmake | 519 ++++++++++ .../binex/config/cmake/HDFPluginMacros.cmake | 279 +++++ .../cmake/binex/config/cmake/cacheinit.cmake | 22 + .../cmake/binex/config/cmake/grepTest.cmake | 199 ++++ .../cmake/binex/config/cmake/runTest.cmake | 387 +++++++ .../binex/config/toolchain/aarch64.cmake | 17 + .../binex/config/toolchain/build32.cmake | 90 ++ .../cmake/binex/config/toolchain/clang.cmake | 27 + .../cmake/binex/config/toolchain/crayle.cmake | 21 + .../cmake/binex/config/toolchain/gcc.cmake | 22 + .../cmake/binex/config/toolchain/icc.cmake | 11 + .../binex/config/toolchain/mingw64.cmake | 25 + .../cmake/binex/config/toolchain/pgi.cmake | 22 + SZ3/config/cmake/binex/example/CMakeLists.txt | 173 ++++ SZ3/config/cmake/binex/example/h5ex_d_sz3.c | 253 +++++ SZ3/config/cmake/binex/example/test/test.sh | 258 +++++ SZ3/config/cmake/binex/example/test/testCM.sh | 258 +++++ .../binex/example/testfiles/h5ex_d_sz3.ddl | 209 ++++ .../binex/example/testfiles/h5ex_d_sz3.tst | 12 + .../example/testfiles/h5repack_layout.h5 | Bin 0 -> 27280 bytes .../h5repack_layout.h5-ud_convert.ddl | 158 +++ .../ud_convert.h5repack_layout.h5.tst | 15 + SZ3/config/cmake/cacheinit.cmake | 29 + SZ3/config/cmake/config.h.in | 91 ++ SZ3/config/cmake/distribution.entitlements | 16 + SZ3/config/cmake/grepTest.cmake | 199 ++++ .../cmake/h5sz3-config-version.cmake.in | 38 + SZ3/config/cmake/h5sz3-config.cmake.in | 70 ++ SZ3/config/cmake/runTest.cmake | 389 +++++++ SZ3/config/toolchain/aarch64.cmake | 18 + SZ3/config/toolchain/build32.cmake | 90 ++ SZ3/config/toolchain/clang.cmake | 27 + SZ3/config/toolchain/crayle.cmake | 21 + SZ3/config/toolchain/gcc.cmake | 22 + SZ3/config/toolchain/icc.cmake | 11 + SZ3/config/toolchain/intel.cmake | 11 + SZ3/config/toolchain/mingw64.cmake | 25 + SZ3/config/toolchain/pgi.cmake | 22 + SZ3/example/CMakeLists.txt | 179 ++++ SZ3/example/h5ex_d_sz3.c | 253 +++++ SZ3/example/test/test.sh | 258 +++++ SZ3/example/test/testCM.sh | 258 +++++ SZ3/example/testfiles/h5ex_d_sz3.ddl | 209 ++++ SZ3/example/testfiles/h5ex_d_sz3.tst | 12 + SZ3/example/testfiles/h5repack_layout.h5 | Bin 0 -> 27280 bytes .../h5repack_layout.h5-ud_convert.ddl | 158 +++ .../ud_convert.h5repack_layout.h5.tst | 15 + docs/PluginLibraries.txt | 35 + docs/RegisteredFilterPlugins.md | 2 +- 83 files changed, 10856 insertions(+), 3 deletions(-) create mode 100644 SZ3/Additional_Legal/copyright-and-BSD-license.txt create mode 100644 SZ3/Additional_Legal/zstd-LICENSE create mode 100644 SZ3/CMakeLists.txt create mode 100644 SZ3/CTestConfig.cmake create mode 100644 SZ3/README.md create mode 100644 SZ3/UPDATING_SZ3_SUBTREE.md create mode 100644 SZ3/config/cmake-presets/hidden-presets.json create mode 100644 SZ3/config/cmake/CTestCustom.cmake create mode 100755 SZ3/config/cmake/CTestScript.cmake create mode 100644 SZ3/config/cmake/ConfigureChecks.cmake create mode 100644 SZ3/config/cmake/H5PLTests.c create mode 100644 SZ3/config/cmake/H5PL_Examples.cmake.in create mode 100644 SZ3/config/cmake/HDFMacros.cmake create mode 100755 SZ3/config/cmake/HDFPLoptions.cmake create mode 100644 SZ3/config/cmake/HDFPluginMacros.cmake create mode 100644 SZ3/config/cmake/NSIS.InstallOptions.ini.in create mode 100644 SZ3/config/cmake/NSIS.template.in create mode 100644 SZ3/config/cmake/README.txt.cmake.in create mode 100644 SZ3/config/cmake/SignPackageFiles.cmake create mode 100644 SZ3/config/cmake/binex/CMakeLists.txt create mode 100644 SZ3/config/cmake/binex/CMakePresets.json create mode 100644 SZ3/config/cmake/binex/CTestConfig.cmake create mode 100644 SZ3/config/cmake/binex/Using_CMake.txt create mode 100644 SZ3/config/cmake/binex/config/cmake-presets/hidden-presets.json create mode 100755 SZ3/config/cmake/binex/config/cmake/CTestCustom.cmake create mode 100644 SZ3/config/cmake/binex/config/cmake/HDFMacros.cmake create mode 100644 SZ3/config/cmake/binex/config/cmake/HDFPluginMacros.cmake create mode 100755 SZ3/config/cmake/binex/config/cmake/cacheinit.cmake create mode 100644 SZ3/config/cmake/binex/config/cmake/grepTest.cmake create mode 100644 SZ3/config/cmake/binex/config/cmake/runTest.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/aarch64.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/build32.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/clang.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/crayle.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/gcc.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/icc.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/mingw64.cmake create mode 100644 SZ3/config/cmake/binex/config/toolchain/pgi.cmake create mode 100644 SZ3/config/cmake/binex/example/CMakeLists.txt create mode 100644 SZ3/config/cmake/binex/example/h5ex_d_sz3.c create mode 100644 SZ3/config/cmake/binex/example/test/test.sh create mode 100755 SZ3/config/cmake/binex/example/test/testCM.sh create mode 100644 SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.ddl create mode 100644 SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.tst create mode 100644 SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5 create mode 100644 SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl create mode 100644 SZ3/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst create mode 100644 SZ3/config/cmake/cacheinit.cmake create mode 100644 SZ3/config/cmake/config.h.in create mode 100644 SZ3/config/cmake/distribution.entitlements create mode 100644 SZ3/config/cmake/grepTest.cmake create mode 100644 SZ3/config/cmake/h5sz3-config-version.cmake.in create mode 100644 SZ3/config/cmake/h5sz3-config.cmake.in create mode 100644 SZ3/config/cmake/runTest.cmake create mode 100644 SZ3/config/toolchain/aarch64.cmake create mode 100644 SZ3/config/toolchain/build32.cmake create mode 100644 SZ3/config/toolchain/clang.cmake create mode 100644 SZ3/config/toolchain/crayle.cmake create mode 100644 SZ3/config/toolchain/gcc.cmake create mode 100644 SZ3/config/toolchain/icc.cmake create mode 100644 SZ3/config/toolchain/intel.cmake create mode 100644 SZ3/config/toolchain/mingw64.cmake create mode 100644 SZ3/config/toolchain/pgi.cmake create mode 100644 SZ3/example/CMakeLists.txt create mode 100644 SZ3/example/h5ex_d_sz3.c create mode 100644 SZ3/example/test/test.sh create mode 100755 SZ3/example/test/testCM.sh create mode 100644 SZ3/example/testfiles/h5ex_d_sz3.ddl create mode 100644 SZ3/example/testfiles/h5ex_d_sz3.tst create mode 100644 SZ3/example/testfiles/h5repack_layout.h5 create mode 100644 SZ3/example/testfiles/h5repack_layout.h5-ud_convert.ddl create mode 100644 SZ3/example/testfiles/ud_convert.h5repack_layout.h5.tst diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 26156a57..58c7e5b9 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -20,4 +20,4 @@ jobs: extensions: 'c,h,cpp,hpp,java' clangFormatVersion: 17 style: file - exclude: './community ./ZFP/H5Z-ZFP' + exclude: './community ./ZFP/H5Z-ZFP ./SZ3/SZ3' diff --git a/.github/workflows/clang-format-fix.yml b/.github/workflows/clang-format-fix.yml index 1717f2f2..94a3500d 100644 --- a/.github/workflows/clang-format-fix.yml +++ b/.github/workflows/clang-format-fix.yml @@ -33,7 +33,7 @@ jobs: clangFormatVersion: 17 inplace: True style: file - exclude: './community ./ZFP/H5Z-ZFP' + exclude: './community ./ZFP/H5Z-ZFP ./SZ3/SZ3' - uses: EndBug/add-and-commit@cc9c08ba6c8df3b93a8f2db63e89b98368ae2ae8 # v11.1.1 with: diff --git a/.github/workflows/main-fetchcontent-mac.yml b/.github/workflows/main-fetchcontent-mac.yml index 6005e279..025f2b76 100644 --- a/.github/workflows/main-fetchcontent-mac.yml +++ b/.github/workflows/main-fetchcontent-mac.yml @@ -84,6 +84,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ .. cmake --build . --parallel 3 --config ${{ inputs.build_mode }} @@ -168,6 +169,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ $GITHUB_WORKSPACE cmake --build . --parallel 3 --config ${{ inputs.build_mode }} @@ -244,6 +246,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ $GITHUB_WORKSPACE cmake --build . --parallel 3 --config ${{ inputs.build_mode }} diff --git a/.github/workflows/main-fetchcontent-ubuntu.yml b/.github/workflows/main-fetchcontent-ubuntu.yml index 9bd862e7..94f36396 100644 --- a/.github/workflows/main-fetchcontent-ubuntu.yml +++ b/.github/workflows/main-fetchcontent-ubuntu.yml @@ -79,6 +79,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ .. # NOTE: Currently a serial build has to be forced, as Blosc and Blosc2 @@ -160,6 +161,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ $GITHUB_WORKSPACE # NOTE: Currently, a serial build has to be forced as Blosc and Blosc2 @@ -238,6 +240,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ $GITHUB_WORKSPACE # NOTE: Currently, a serial build has to be forced as Blosc and Blosc2 diff --git a/.github/workflows/main-fetchcontent-win.yml b/.github/workflows/main-fetchcontent-win.yml index c0a54793..fcbd9c43 100644 --- a/.github/workflows/main-fetchcontent-win.yml +++ b/.github/workflows/main-fetchcontent-win.yml @@ -88,6 +88,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ .. cmake --build . --parallel 4 --config ${{ inputs.build_mode }} @@ -177,6 +178,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ $GITHUB_WORKSPACE cmake --build . --parallel 4 --config ${{ inputs.build_mode }} @@ -264,6 +266,7 @@ jobs: -DENABLE_BITGROOM=ON \ -DENABLE_GRANULAR_BITROUND=ON \ -DENABLE_BITROUND=ON \ + -DENABLE_SZ3=ON \ -DENABLE_BSHUF=ON \ $GITHUB_WORKSPACE cmake --build . --parallel 4 --config ${{ inputs.build_mode }} diff --git a/.github/workflows/main-package-managers-mac.yml b/.github/workflows/main-package-managers-mac.yml index 749e9366..f54bc444 100644 --- a/.github/workflows/main-package-managers-mac.yml +++ b/.github/workflows/main-package-managers-mac.yml @@ -124,6 +124,7 @@ jobs: -DENABLE_BITGROOM=OFF \ -DENABLE_GRANULAR_BITROUND=OFF \ -DENABLE_BITROUND=OFF \ + -DENABLE_SZ3=OFF \ -DENABLE_BSHUF=OFF \ $GITHUB_WORKSPACE cmake --build . --parallel 3 --config ${{ inputs.build_mode }} diff --git a/.github/workflows/main-package-managers-ubuntu.yml b/.github/workflows/main-package-managers-ubuntu.yml index 20244c50..d4ec4d47 100644 --- a/.github/workflows/main-package-managers-ubuntu.yml +++ b/.github/workflows/main-package-managers-ubuntu.yml @@ -98,6 +98,7 @@ jobs: -DENABLE_BITGROOM=OFF \ -DENABLE_GRANULAR_BITROUND=OFF \ -DENABLE_BITROUND=OFF \ + -DENABLE_SZ3=OFF \ -DENABLE_BLOSC=OFF \ -DENABLE_BSHUF=OFF \ -DENABLE_BZIP2=OFF \ diff --git a/.github/workflows/main-package-managers-win.yml b/.github/workflows/main-package-managers-win.yml index 457e87cb..9227f4cf 100644 --- a/.github/workflows/main-package-managers-win.yml +++ b/.github/workflows/main-package-managers-win.yml @@ -147,6 +147,7 @@ jobs: -DENABLE_BITGROOM=OFF \ -DENABLE_GRANULAR_BITROUND=OFF \ -DENABLE_BITROUND=OFF \ + -DENABLE_SZ3=OFF \ -DENABLE_BSHUF=OFF \ $GITHUB_WORKSPACE cmake --build . --parallel 4 --config ${{ inputs.build_mode }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 7dc294f4..ecd7721d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,6 +187,14 @@ else () set (ENABLE_LZF OFF CACHE BOOL "" FORCE) endif () +# SZ3 filter +# SZ3 has no decode-only build, and its CMake requires CMake 3.19 +if (NOT MINGW AND NOT DISABLE_H5PL_ENCODER AND CMAKE_VERSION VERSION_GREATER_EQUAL "3.19") + FILTER_OPTION (SZ3) +else () + set (ENABLE_SZ3 OFF CACHE BOOL "" FORCE) +endif () + # ZFP filter if (WIN32) if (NOT CMAKE_C_COMPILER_ID MATCHES "Intel") diff --git a/SZ3/Additional_Legal/copyright-and-BSD-license.txt b/SZ3/Additional_Legal/copyright-and-BSD-license.txt new file mode 100644 index 00000000..e48a51c5 --- /dev/null +++ b/SZ3/Additional_Legal/copyright-and-BSD-license.txt @@ -0,0 +1,29 @@ +Copyright © 2016 , UChicago Argonne, LLC +All Rights Reserved + +SZ licence is under the BSD license. Details are shown below. + +OPEN SOURCE LICENSE (license number: SF-16-105) + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Software changes, modifications, or derivative works, should be noted with comments and the author and organization's name. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the names of UChicago Argonne, LLC or the Department of Energy nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +4. The software and the end-user documentation included with the redistribution, if any, must include the following acknowledgment: + + "This product includes software produced by UChicago Argonne, LLC under Contract No. DE-AC02-06CH11357 with the Department of Energy." + +****************************************************************************************************** +DISCLAIMER + +THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. + +NEITHER THE UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS. + +*************************************************************************************************** + +Contact: Sheng Di (sdi1@anl.gov), Franck Cappello(cappello@mcs.anl.gov) diff --git a/SZ3/Additional_Legal/zstd-LICENSE b/SZ3/Additional_Legal/zstd-LICENSE new file mode 100644 index 00000000..75800288 --- /dev/null +++ b/SZ3/Additional_Legal/zstd-LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/SZ3/CMakeLists.txt b/SZ3/CMakeLists.txt new file mode 100644 index 00000000..aee791d3 --- /dev/null +++ b/SZ3/CMakeLists.txt @@ -0,0 +1,204 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +# Wrapper CMakeLists.txt for SZ3 subtree integration +# This integrates the upstream H5Z-SZ3 filter (tools/H5Z-SZ3 of SZ3, via git +# subtree) with the HDF Group's hdf5_plugins build system and testing infrastructure. + +cmake_minimum_required(VERSION 3.18) +project(H5SZ3 C CXX) + +if (POLICY CMP0074) + # find_package() uses _ROOT variables. + cmake_policy(SET CMP0074 NEW) +endif () + +if (POLICY CMP0083) + # To control generation of Position Independent Executable (PIE) or not, + # some flags are required at link time. + cmake_policy(SET CMP0083 NEW) +endif () + +# Avoid warning about DOWNLOAD_EXTRACT_TIMESTAMP in CMake 3.24: +if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") + cmake_policy(SET CMP0135 NEW) +endif() + +#----------------------------------------------------------------------------- +# Verify SZ3 subtree exists +#----------------------------------------------------------------------------- +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/SZ3/tools/H5Z-SZ3/CMakeLists.txt") + message(FATAL_ERROR + "SZ3 not found at ${CMAKE_CURRENT_SOURCE_DIR}/SZ3\n" + "The upstream SZ3 code should be present via git subtree.") +endif() + +# SZ3's CMake requires CMake 3.19 +if (CMAKE_VERSION VERSION_LESS "3.19") + message(FATAL_ERROR "The SZ3 filter plugin requires CMake 3.19 or newer") +endif () + +message(STATUS "Using SZ3 from subtree at ${CMAKE_CURRENT_SOURCE_DIR}/SZ3") + +#----------------------------------------------------------------------------- +# Basic H5SZ3 stuff - integrate with HDF Group build system +#----------------------------------------------------------------------------- +if (NOT H5PL_RESOURCES_DIR) + include(${H5SZ3_SOURCE_DIR}/config/cmake/HDFMacros.cmake) + include(${H5SZ3_SOURCE_DIR}/config/cmake/HDFPluginMacros.cmake) + + SET_HDF_BUILD_TYPE() +endif () + +# Set up package name for HDF Group infrastructure +if (NOT H5SZ3_PACKAGE_NAME) + set(H5SZ3_PACKAGE_NAME "sz3") +endif() + +BASIC_SETTINGS(H5${H5SZ3_PACKAGE_NAME}) + +# The H5Z-SZ3 filter has no decode-only build +if (DISABLE_H5SZ3_ENCODER) + message(FATAL_ERROR "The ${H5SZ3_PACKAGE_NAME} filter cannot be built without its encoder; set ENABLE_SZ3=OFF instead") +endif () + +#----------------------------------------------------------------------------- +# Version information (read from SZ3) +#----------------------------------------------------------------------------- +file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/SZ3/CMakeLists.txt" SZ3_PROJECT_LINE REGEX "^project\\(SZ3 VERSION ") +string(REGEX MATCH "([0-9]+)\\.([0-9]+)\\.([0-9]+)" SZ3_PROJECT_VERSION "${SZ3_PROJECT_LINE}") +set(H5SZ3_VERS_MAJOR ${CMAKE_MATCH_1}) +set(H5SZ3_VERS_MINOR ${CMAKE_MATCH_2}) +set(H5SZ3_VERS_RELEASE ${CMAKE_MATCH_3}) + +set(H5SZ3_PACKAGE_VERSION "${H5SZ3_VERS_MAJOR}.${H5SZ3_VERS_MINOR}.${H5SZ3_VERS_RELEASE}") +set(H5SZ3_PACKAGE_VERSION_STRING "${H5SZ3_PACKAGE_VERSION}") +set(H5SZ3_PACKAGE_VERSION_MAJOR "${H5SZ3_VERS_MAJOR}") +set(H5SZ3_PACKAGE_VERSION_MINOR "${H5SZ3_VERS_MINOR}") +set(H5SZ3_PACKAGE_STRING "${H5SZ3_PACKAGE_NAME} ${H5SZ3_PACKAGE_VERSION}") +set(H5SZ3_PACKAGE_URL "http://www.hdfgroup.org") +set(H5SZ3_PACKAGE_BUGREPORT "help@hdfgroup.org") + +message(STATUS "Configuring SZ3 HDF5 Plugin version: ${H5SZ3_PACKAGE_STRING} (from upstream SZ3)") + +#----------------------------------------------------------------------------- +# Include HDF Group macros +#----------------------------------------------------------------------------- +include(${H5SZ3_RESOURCES_DIR}/ConfigureChecks.cmake) + +#----------------------------------------------------------------------------- +# HDF5 support +#----------------------------------------------------------------------------- +HDF5_SUPPORT(TRUE) +message(STATUS "H5SZ3 link libs: ${H5PL_LINK_LIBS}") + +#----------------------------------------------------------------------------- +# Add the upstream H5Z-SZ3 filter via SZ3's own CMake +# Note: SZ3 is a header-only C++17 library; with BUILD_H5Z_FILTER its +# CMakeLists.txt builds the filter plugin (target hdf5sz3) from tools/H5Z-SZ3 +#----------------------------------------------------------------------------- +message(STATUS "Building H5Z-SZ3 filter from subtree") + +# Build the filter, and none of SZ3's tools or tests +set(BUILD_H5Z_FILTER ON) +set(BUILD_SZ3_BINARY OFF) +set(BUILD_MDZ OFF) +set(BUILD_PARAVIEW_PLUGIN OFF) + +# Compile the Zstd bundled in SZ3 (SZ3/tools/zstd) into the plugin, so the +# plugin needs no Zstd library at build or run time +set(SZ3_USE_BUNDLED_ZSTD ON) + +# SZ3 installs the filter plugin into H5Z_SZ3_PLUGIN_INSTALL_DIR (default +# lib/plugin) only when SZ3_INSTALL is ON, which it is not by default under a +# parent project. Point it at this project's lib/plugin convention +# (H5SZ3_INSTALL_LIB_DIR, set above by BASIC_SETTINGS). +set(SZ3_INSTALL ON) +set(H5Z_SZ3_PLUGIN_INSTALL_DIR ${H5SZ3_INSTALL_LIB_DIR} CACHE STRING + "Directory to install the HDF5 filter plugin copy into; empty installs no copy") +add_subdirectory(SZ3) + +# Copy the plugin to the global plugins directory for testing +# SZ3 builds it in its own binary dir, but tests expect plugins in build/plugins/ +# Use a custom target since add_custom_command(TARGET) only works in same directory +add_custom_target(copy_sz3_plugin ALL + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/plugins" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" + "${CMAKE_BINARY_DIR}/plugins/$" + DEPENDS hdf5sz3 + COMMENT "Copying SZ3 plugin to ${CMAKE_BINARY_DIR}/plugins/" +) + +#----------------------------------------------------------------------------- +# Create compatibility alias for the HDF Group target name +#----------------------------------------------------------------------------- +# The HDF Group build names plugin targets ${H5SZ3_LIB_TARGET} (h5sz3) +# SZ3 creates hdf5sz3, so create an alias for compatibility +if (TARGET hdf5sz3) + set(H5SZ3_LIB_CORENAME "h5sz3") + set(H5SZ3_LIB_TARGET ${H5SZ3_LIB_CORENAME}) + + # Create alias so parent build system can find it by the HDF Group name + add_library(${H5SZ3_LIB_TARGET} ALIAS hdf5sz3) + + message(STATUS "Created alias: ${H5SZ3_LIB_TARGET} -> hdf5sz3") +endif () + +#----------------------------------------------------------------------------- +# Export the library for parent build system +#----------------------------------------------------------------------------- +# Export the expected target name for the parent build system +set(H5SZ3_LIBRARIES_TO_EXPORT "${H5SZ3_LIB_TARGET}" CACHE STRING "H5SZ3 libraries to export" FORCE) + +#----------------------------------------------------------------------------- +# The SZ3 and Zstd licenses ask binary redistributions to carry their notices +#----------------------------------------------------------------------------- +install( + FILES + ${H5SZ3_SOURCE_DIR}/Additional_Legal/copyright-and-BSD-license.txt + ${H5SZ3_SOURCE_DIR}/Additional_Legal/zstd-LICENSE + DESTINATION ${H5SZ3_INSTALL_DOC_DIR}/SZ3 + COMPONENT libraries +) + +#----------------------------------------------------------------------------- +# Dashboard and Testing Settings +#----------------------------------------------------------------------------- +option(H5PL_BUILD_TESTING "Build h5sz3 Unit Testing" OFF) +if (H5PL_BUILD_TESTING) + set(DART_TESTING_TIMEOUT 1200 CACHE STRING + "Timeout in seconds for each test (default 1200=20minutes)") + enable_testing() + include(CTest) + include(${PROJECT_SOURCE_DIR}/CTestConfig.cmake) + configure_file(${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.ctest @ONLY) +endif () + +#----------------------------------------------------------------------------- +# Add HDF Group examples +#----------------------------------------------------------------------------- +# Examples are needed for testing, so enable them if testing is enabled +if (H5PL_BUILD_TESTING AND NOT H5PL_BUILD_EXAMPLES) + set(H5PL_BUILD_EXAMPLES ON CACHE BOOL "Build h5sz3 Examples (required for testing)" FORCE) + message(STATUS "Enabling H5PL_BUILD_EXAMPLES (required for H5PL_BUILD_TESTING)") +endif () + +option(H5PL_BUILD_EXAMPLES "Build h5sz3 Examples" OFF) +if (H5PL_BUILD_EXAMPLES) + add_subdirectory(example) +endif () + +#----------------------------------------------------------------------------- +# Add file(s) to CMake Install +#----------------------------------------------------------------------------- +INSTALL_SUPPORT(H5SZ3) diff --git a/SZ3/CTestConfig.cmake b/SZ3/CTestConfig.cmake new file mode 100644 index 00000000..b1a3f2ce --- /dev/null +++ b/SZ3/CTestConfig.cmake @@ -0,0 +1,36 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +## This file should be placed in the root directory of your project. +## Then modify the CMakeLists.txt file in the root directory of your +## project to incorporate the testing dashboard. +## # The following are required to uses Dart and the Cdash dashboard +## ENABLE_TESTING() +## INCLUDE(CTest) +set (CTEST_PROJECT_NAME "HDF5PLUGINS") +set (CTEST_NIGHTLY_START_TIME "18:00:00 CST") + +set (CTEST_DROP_METHOD "https") +set (CTEST_DROP_SITE "cdash.hdfgroup.org") +set (CTEST_DROP_LOCATION "/submit.php?project=HDF5PLUGINS") +set (CTEST_DROP_SITE_CDASH TRUE) + +set (UPDATE_TYPE git) +set (VALGRIND_COMMAND "/usr/bin/valgrind") +set (VALGRIND_COMMAND_OPTIONS "-v --tool=memcheck --leak-check=full --track-fds=yes --num-callers=50 --show-reachable=yes --track-origins=yes --malloc-fill=0xff --free-fill=0xfe") + +set (CTEST_TEST_TIMEOUT 3600 CACHE STRING + "Maximum time allowed before CTest will kill the test.") +set (DART_TESTING_TIMEOUT 3600 CACHE STRING + "Maximum time allowed before CTest will kill the test." FORCE) + +set (CTEST_SUBMIT_RETRY_DELAY 20 CACHE STRING + "How long to wait between timed-out CTest submissions.") diff --git a/SZ3/README.md b/SZ3/README.md new file mode 100644 index 00000000..bb1a2727 --- /dev/null +++ b/SZ3/README.md @@ -0,0 +1,129 @@ +# H5Z-SZ3 Filter Plugin (ID 32024) + +This directory integrates the H5Z-SZ3 filter plugin from the SZ3 project using **git subtree**. + +## Upstream Source + +- **Repository**: https://github.com/szcompressor/SZ3 (the filter is in `tools/H5Z-SZ3`) +- **Version**: v3.4.0 +- **License**: BSD-style (see SZ3/copyright-and-BSD-license.txt); the bundled Zstd is BSD 3-Clause (see SZ3/tools/zstd/LICENSE) +- **Maintainer**: Kai Zhao (lead developer of SZ3) +- **Documentation**: https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + +## About H5Z-SZ3 + +SZ3 is a modular error-bounded lossy compression framework for scientific datasets. +H5Z-SZ3 compresses float, double and 1, 2, 4 or 8-byte integer datasets in host byte +order, with chunks of up to 4 dimensions longer than 1, within a user-set error bound: +- **ABS**: Limit absolute error +- **REL**: Limit error relative to the value range +- **PSNR**: Target a peak signal-to-noise ratio +- **L2NORM**: Limit the L2 norm of the error + +Without `cd_values` the filter uses SZ3's default settings, the `ALGO_INTERP_LORENZO` +algorithm with an absolute error bound of 1e-3: + +```c +H5Pset_chunk(dcpl, rank, chunk); +H5Pset_filter(dcpl, 32024, H5Z_FLAG_MANDATORY, 0, NULL); +``` + +```sh +h5repack -f UD=32024,0 in.h5 out.h5 +``` + +`cd_values[0]` is the SZ3 data format version, followed by the SZ3 configuration +serialized by SZ3. To choose the algorithm and error bound, use `H5Pset_sz3()` from the +H5Z-SZ3 library, or `cdvalueHelper` from SZ3 to print the `cd_values` for `h5repack`. +Decompression reads the settings from the compressed data. + +## Building + +### No Special Setup Required + +The SZ3 source, including the H5Z-SZ3 filter and the Zstd library bundled in SZ3, is +included directly in this repository via git subtree. Simply clone and build: + +```bash +git clone https://github.com/HDFGroup/hdf5_plugins.git +# Everything is ready - no submodule initialization needed! +``` + +### Build Requirements + +H5Z-SZ3 requires: +1. **HDF5 library** - Set `HDF5_ROOT` environment variable +2. **A C++17 compiler** and **CMake 3.19 or newer** - SZ3 is a header-only C++17 library + +No external compression library is needed: SZ3 is header-only, and the wrapper builds +SZ3's bundled Zstd (`SZ3_USE_BUNDLED_ZSTD`) into the plugin. The filter therefore builds +with any `H5PL_ALLOW_EXTERNAL_SUPPORT` setting, including `NO`. It is not built on MinGW +or with `DISABLE_H5PL_ENCODER`, as H5Z-SZ3 has no decode-only build. + +### Building as Part of hdf5_plugins + +```bash +cd hdf5_plugins +export HDF5_ROOT=/path/to/hdf5 +mkdir build && cd build + +cmake -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_SZ3=ON \ + .. + +cmake --build . +``` + +The plugin is SZ3's `hdf5sz3` target (`libhdf5sz3.so`, `hdf5sz3.dll`), aliased as `h5sz3`. +It is copied to `build/plugins` for testing and installed to `lib/plugin` +(`H5Z_SZ3_PLUGIN_INSTALL_DIR`). As SZ3's own install, the SZ3 headers, the H5Z-SZ3 header +and library, and SZ3's CMake package are installed too. The SZ3 and Zstd license notices +are installed with the plugin. + +### Building Standalone + +The directory can also be configured on its own: + +```bash +cmake -S hdf5_plugins/SZ3 -B build-sz3 -DCMAKE_BUILD_TYPE=Release -DH5PL_BUILD_TESTING=ON +cmake --build build-sz3 +``` + +## Updating to Newer Versions (Maintainers Only) + +HDF Group maintainers can update to newer SZ3 versions using git subtree: + +```bash +# One-time: Add upstream remote (if not already done) +git remote add sz3 https://github.com/szcompressor/SZ3.git + +# Update to specific version +git subtree pull --prefix=SZ3/SZ3 sz3 v3.4.1 --squash +``` + +**Important**: The `--squash` flag is recommended to avoid importing full upstream history into hdf5_plugins. + +See [UPDATING_SZ3_SUBTREE.md](UPDATING_SZ3_SUBTREE.md) for the full procedure. + +## Reporting Issues + +### Filter Issues +Issues with the SZ3 filter itself (bugs, feature requests, compression behavior) should be reported at: +**https://github.com/szcompressor/SZ3/issues** + +### Integration Issues +Issues with how H5Z-SZ3 integrates with the hdf5_plugins build system should be reported at: +**https://github.com/HDFGroup/hdf5_plugins/issues** + +## Policy: Upstream-First Development + +HDF Group does not maintain a fork of SZ3. This directory uses the upstream repository directly via git subtree. + +**Important**: Never modify files in `SZ3/SZ3/` directly! + +If you need changes to the filter or its build: +1. Open an issue at https://github.com/szcompressor/SZ3/issues +2. Submit a pull request to upstream +3. After upstream merges, use `git subtree pull` to update (see above) + +This ensures the HDF community benefits from improvements and the upstream maintainers remain authoritative. diff --git a/SZ3/UPDATING_SZ3_SUBTREE.md b/SZ3/UPDATING_SZ3_SUBTREE.md new file mode 100644 index 00000000..de84c7a3 --- /dev/null +++ b/SZ3/UPDATING_SZ3_SUBTREE.md @@ -0,0 +1,193 @@ +# Updating the SZ3 Subtree + +This document explains how to update the H5Z-SZ3 filter plugin (SZ3) to a newer upstream version. + +## Overview + +The SZ3 filter is integrated using **git subtree**, which embeds the upstream szcompressor/SZ3 repository directly into `SZ3/SZ3/`. The filter itself is SZ3's `tools/H5Z-SZ3`, built by SZ3's own CMake from the wrapper `SZ3/CMakeLists.txt`. This allows users to clone and build without any additional setup while maintaining a clear connection to upstream. + +## Prerequisites + +Before updating, ensure: +1. You have a clean working tree: `git status` shows no uncommitted changes +2. You're on the appropriate branch (usually `master` or a feature branch) +3. You have push access to the hdf5_plugins repository + +## Step-by-Step Update Process + +### 1. Add Upstream Remote (One-Time Setup) + +If you haven't already added the upstream SZ3 remote: + +```bash +git remote add sz3 https://github.com/szcompressor/SZ3.git +``` + +Verify the remote was added: + +```bash +git remote -v +``` + +You should see `sz3` pointing to the szcompressor repository. + +### 2. Fetch Upstream Changes + +Before pulling, fetch the latest upstream changes: + +```bash +git fetch sz3 --tags --force +``` + +### 3. Check Available Versions + +List available upstream tags to see what versions are available: + +```bash +git ls-remote --tags sz3 +``` + +### 4. Pull the Update + +Update to a release tag (recommended for stability): + +```bash +git subtree pull --prefix=SZ3/SZ3 sz3 v3.4.1 --squash -m "Update SZ3 to v3.4.1" +``` + +Replace `v3.4.1` with the desired version tag. Check that the commit the tag points to (`git ls-remote --tags sz3 v3.4.1`) is the one you expect, and +compare it with the `git-subtree-split:` line of the squash commit after the pull. + +### 5. Verify the Update + +After pulling, verify the update was successful: + +```bash +# Check that SZ3/SZ3 directory contains the new code +ls -la SZ3/SZ3/ + +# View the commit created by subtree pull +git log -1 + +# Check git status +git status +``` + +### 6. Test the Build + +Before pushing, ensure the updated filter builds correctly: + +```bash +# Set HDF5_ROOT to your HDF5 installation +export HDF5_ROOT=/path/to/hdf5 + +# Create a clean build directory +rm -rf build +mkdir build && cd build + +# Configure with SZ3 enabled (no external libraries are needed) +cmake -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_SZ3=ON \ + -DH5PL_ALLOW_EXTERNAL_SUPPORT=NO \ + -DH5PL_BUILD_TESTING=ON \ + .. + +# Build +cmake --build . + +# Run tests +ctest -R "SZ3|sz3" +``` + +### 7. Commit and Push + +If the build and tests pass: + +```bash +# The subtree pull already created a commit, so just push +git push origin +``` + +## Understanding the --squash Flag + +The `--squash` flag is **recommended** for subtree updates. Here's why: + +- **Without --squash**: Imports the entire upstream commit history into hdf5_plugins + - Pro: Preserves full upstream history + - Con: Makes hdf5_plugins history cluttered with upstream commits + +- **With --squash**: Combines all upstream changes into a single commit + - Pro: Keeps hdf5_plugins history clean + - Con: Loses individual upstream commit details (but you can still view them in the upstream repo) + +**Recommendation**: Always use `--squash` unless you have a specific reason to preserve full upstream history. + +## Troubleshooting + +### Merge Conflicts + +If you encounter merge conflicts during `git subtree pull`: + +1. **Resolve conflicts**: Edit the conflicting files in `SZ3/SZ3/` +2. **Stage resolved files**: `git add SZ3/SZ3/` +3. **Complete the merge**: `git commit` + +**Note**: Conflicts are rare if you never modify files in `SZ3/SZ3/` directly (which is the policy). + +### Checking Current Upstream Version + +To see what upstream commit the current subtree is based on: + +```bash +git log --grep="git-subtree-dir: SZ3/SZ3" --format="%H %s%n%b" -1 +``` + +Look for the commit hash after `git-subtree-split:` in the commit message. + +### Reverting a Failed Update + +If an update fails or causes issues: + +```bash +# Find the commit hash before the subtree pull +git log --oneline -5 + +# Reset to before the update +git reset --hard +``` + +## Update Checklist + +Before pushing an SZ3 update to the main repository: + +- [ ] Upstream remote is configured correctly +- [ ] Fetched latest upstream changes +- [ ] Pulled the desired version using `--squash` +- [ ] Verified `SZ3/SZ3/` contains expected changes +- [ ] `git-subtree-split:` names the upstream commit of the intended tag +- [ ] License notices in `SZ3/Additional_Legal/` still match `SZ3/SZ3/copyright-and-BSD-license.txt` and `SZ3/SZ3/tools/zstd/LICENSE` +- [ ] `docs/PluginLibraries.txt` names the new SZ3 version +- [ ] Build succeeds with `-DENABLE_SZ3=ON` +- [ ] SZ3 tests pass: `ctest -R "SZ3|sz3"` +- [ ] Commit message clearly indicates version/branch updated +- [ ] Ready to push to hdf5_plugins repository + +## Policy Reminder + +**Never modify files in `SZ3/SZ3/` directly!** + +If you need changes to the H5Z-SZ3 filter or to SZ3's build: +1. Open an issue at https://github.com/szcompressor/SZ3/issues +2. Submit a pull request to upstream +3. After upstream merges, use `git subtree pull` to update + +This ensures: +- The HDF community benefits from improvements +- Upstream maintainers remain authoritative +- Updates remain straightforward without merge conflicts + +## Additional Resources + +- H5Z-SZ3 Documentation: https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 +- SZ3 Repository: https://github.com/szcompressor/SZ3 +- Git Subtree Documentation: `man git-subtree` or https://git-scm.com/docs/git-subtree diff --git a/SZ3/config/cmake-presets/hidden-presets.json b/SZ3/config/cmake-presets/hidden-presets.json new file mode 100644 index 00000000..c78b4c4e --- /dev/null +++ b/SZ3/config/cmake-presets/hidden-presets.json @@ -0,0 +1,514 @@ +{ + "version": 6, + "configurePresets": [ + { + "name": "ci-base", + "displayName": "Basic Config", + "description": "Basic build using Ninja generator", + "generator": "Ninja", + "hidden": true, + "binaryDir": "${sourceParentDir}/build/${presetName}", + "installDir": "${sourceParentDir}/install/${presetName}" + }, + { + "name": "ci-x64", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-x86", + "architecture": { + "value": "Win32", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-arm64", + "architecture": { + "value": "ARM64", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-Debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "hidden": true + }, + { + "name": "ci-Release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + }, + "hidden": true + }, + { + "name": "ci-MSVC", + "hidden": true, + "cacheVariables": { + "CMAKE_C_COMPILER": "cl", + "CMAKE_CXX_COMPILER": "cl" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "ci-macos-arm64", + "hidden": true, + "cacheVariables": { + "CMAKE_OSX_ARCHITECTURES": "arm64" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + } + }, + { + "name": "ci-macos-x86_64", + "hidden": true, + "cacheVariables": { + "CMAKE_OSX_ARCHITECTURES": "x86_64" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + } + }, + { + "name": "ci-Clang", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/clang.cmake" + } + }, + { + "name": "ci-GNUC", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/gcc.cmake" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Linux" + } + }, + { + "name": "ci-Intel", + "hidden": true + }, + { + "name": "ci-x64-Debug-MSVC", + "description": "MSVC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-MSVC" + ] + }, + { + "name": "ci-x64-Release-MSVC", + "description": "MSVC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-MSVC" + ] + }, + { + "name": "ci-x64-Debug-Clang", + "description": "Clang/LLVM for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-Clang" + ] + }, + { + "name": "ci-x64-Release-Clang", + "description": "Clang/LLVM for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-Clang" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "description": "GNUC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-GNUC" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "description": "GNUC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-GNUC" + ] + }, + { + "name": "ci-macos-arm64-Debug-Clang", + "description": "Clang/LLVM for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Debug", + "ci-Clang" + ] + }, + { + "name": "ci-macos-arm64-Release-Clang", + "description": "Clang/LLVM for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Release", + "ci-Clang" + ] + }, + { + "name": "ci-macos-arm64-Debug-GNUC", + "description": "GNUC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Debug", + "ci-GNUC" + ] + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "description": "GNUC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Release", + "ci-GNUC" + ] + }, + { + "name": "ci-x64-Debug-Intel", + "description": "Intel for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-Intel" + ] + }, + { + "name": "ci-x64-Release-Intel", + "description": "Intel for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-Intel" + ] + } + ], + "buildPresets": [ + { + "name": "ci-base", + "configurePreset": "ci-base", + "hidden": true, + "verbose": true, + "jobs": 8 + }, + { + "name": "ci-x64-Debug-MSVC", + "configurePreset": "ci-x64-Debug-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "Debug" + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "RelWithDebInfo" + }, + { + "name": "ci-x64-Debug-Clang", + "configurePreset": "ci-x64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "configurePreset": "ci-x64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Debug-Clang", + "configurePreset": "ci-macos-arm64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-Clang", + "configurePreset": "ci-macos-arm64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Debug-GNUC", + "configurePreset": "ci-macos-arm64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "configurePreset": "ci-macos-arm64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-Intel", + "configurePreset": "ci-x64-Debug-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "Debug" + }, + { + "name": "ci-x64-Release-Intel", + "configurePreset": "ci-x64-Release-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "RelWithDebInfo" + } + ], + "testPresets": [ + { + "name": "ci-base", + "configurePreset": "ci-base", + "output": { + "outputOnFailure": false, + "shortProgress": true, + "verbosity": "verbose" + }, + "hidden": true, + "execution": { + "noTestsAction": "error", + "timeout": 600, + "jobs": 8 + } + }, + { + "name": "ci-x64-Debug-MSVC", + "configurePreset": "ci-x64-Debug-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "Debug" + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "RelWithDebInfo" + }, + { + "name": "ci-x64-Debug-Clang", + "configurePreset": "ci-x64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "configurePreset": "ci-x64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-Debug-Clang", + "configurePreset": "ci-macos-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-Clang", + "configurePreset": "ci-macos-arm64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Debug-GNUC", + "configurePreset": "ci-macos-arm64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "configurePreset": "ci-macos-arm64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-Intel", + "configurePreset": "ci-x64-Debug-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Intel", + "configurePreset": "ci-x64-Release-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ] + } + ], + "packagePresets": [ + { + "name": "ci-base", + "hidden": true, + "output": { + "verbose": true + } + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": "ci-base", + "configurations": ["RelWithDebInfo"] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-macos-arm64-Release-Clang", + "configurePreset": "ci-macos-arm64-Release-Clang", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "configurePreset": "ci-macos-arm64-Release-GNUC", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-x64-Release-Intel", + "configurePreset": "ci-x64-Release-Intel", + "hidden": true, + "inherits": "ci-base" + } + ] +} diff --git a/SZ3/config/cmake/CTestCustom.cmake b/SZ3/config/cmake/CTestCustom.cmake new file mode 100644 index 00000000..bcb80c1f --- /dev/null +++ b/SZ3/config/cmake/CTestCustom.cmake @@ -0,0 +1,23 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +set (CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 1500) + +set (CTEST_CUSTOM_WARNING_EXCEPTION + ${CTEST_CUSTOM_WARNING_EXCEPTION} + "POSIX name for this item is deprecated" + "disabling jobserver mode" + "config.cmake.xlatefile.c" +) + +set (CTEST_CUSTOM_MEMCHECK_IGNORE + ${CTEST_CUSTOM_MEMCHECK_IGNORE} +) diff --git a/SZ3/config/cmake/CTestScript.cmake b/SZ3/config/cmake/CTestScript.cmake new file mode 100755 index 00000000..483afd84 --- /dev/null +++ b/SZ3/config/cmake/CTestScript.cmake @@ -0,0 +1,219 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +cmake_minimum_required (VERSION 3.18) +######################################################## +# For any comments please contact help@hdfgroup.org +# +######################################################## +# ----------------------------------------------------------- +# -- Get environment +# ----------------------------------------------------------- +if (NOT SITE_OS_NAME) + ## machine name not provided - attempt to discover with uname + ## -- set hostname + ## -------------------------- + find_program (HOSTNAME_CMD NAMES hostname) + execute_process (COMMAND ${HOSTNAME_CMD} OUTPUT_VARIABLE HOSTNAME OUTPUT_STRIP_TRAILING_WHITESPACE) + set (CTEST_SITE "${HOSTNAME}${CTEST_SITE_EXT}") + find_program (UNAME NAMES uname) + macro (getuname name flag) + execute_process (COMMAND "${UNAME}" "${flag}" OUTPUT_VARIABLE "${name}" OUTPUT_STRIP_TRAILING_WHITESPACE) + endmacro () + + getuname (osname -s) + string(STRIP ${osname} osname) + getuname (osrel -r) + string(STRIP ${osrel} osrel) + getuname (cpu -m) + string(STRIP ${cpu} cpu) + message (STATUS "Dashboard script uname output: ${osname}-${osrel}-${cpu}\n") + + set (CTEST_BUILD_NAME "${osname}-${osrel}-${cpu}") +else () + ## machine name provided + ## -------------------------- + if (CMAKE_HOST_UNIX) + set (CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_OS_BITS}-${SITE_COMPILER_NAME}-${SITE_COMPILER_VERSION}") + else () + set (CTEST_BUILD_NAME "${SITE_OS_NAME}-${SITE_OS_VERSION}-${SITE_COMPILER_NAME}") + endif () +endif () +if (SITE_BUILDNAME_SUFFIX) + set (CTEST_BUILD_NAME "${SITE_BUILDNAME_SUFFIX}-${CTEST_BUILD_NAME}") +endif () +set (BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}") + +# Launchers work only with Makefile and Ninja generators. +if (NOT "${CTEST_CMAKE_GENERATOR}" MATCHES "Make|Ninja" OR LOCAL_SKIP_TEST) + set (CTEST_USE_LAUNCHERS 0) + set (ENV{CTEST_USE_LAUNCHERS_DEFAULT} 0) + set (BUILD_OPTIONS "${BUILD_OPTIONS} -DCTEST_USE_LAUNCHERS:BOOL=OFF") +else () + set (CTEST_USE_LAUNCHERS 1) + set (ENV{CTEST_USE_LAUNCHERS_DEFAULT} 1) + set (BUILD_OPTIONS "${BUILD_OPTIONS} -DCTEST_USE_LAUNCHERS:BOOL=ON") +endif () + +#----------------------------------------------------------------------------- +# MacOS machines need special options +#----------------------------------------------------------------------------- +if (APPLE) + # Compiler choice + execute_process (COMMAND xcrun --find cc OUTPUT_VARIABLE XCODE_CC OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process (COMMAND xcrun --find c++ OUTPUT_VARIABLE XCODE_CXX OUTPUT_STRIP_TRAILING_WHITESPACE) + set (ENV{CC} "${XCODE_CC}") + set (ENV{CXX} "${XCODE_CXX}") + + set (BUILD_OPTIONS "${BUILD_OPTIONS} -DCTEST_USE_LAUNCHERS:BOOL=ON -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=OFF") +endif () + +set (CTEST_CMAKE_COMMAND "\"${CMAKE_COMMAND}\"") +#----------------------------------------------------------------------------- +## Clear the build directory +## -------------------------- +set (CTEST_START_WITH_EMPTY_BINARY_DIRECTORY TRUE) +if (NOT EXISTS "${CTEST_BINARY_DIRECTORY}") + file (MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") +else () + ctest_empty_binary_directory (${CTEST_BINARY_DIRECTORY}) +endif () + +# Use multiple CPU cores to build +include (ProcessorCount) +ProcessorCount (N) +if (NOT N EQUAL 0) + if (MAX_PROC_COUNT) + if (N GREATER MAX_PROC_COUNT) + set (N ${MAX_PROC_COUNT}) + endif () + endif () + if (NOT WIN32) + set (CTEST_BUILD_FLAGS -j${N}) + endif () + set (ctest_test_args ${ctest_test_args} PARALLEL_LEVEL ${N}) +endif () + +#----------------------------------------------------------------------------- +# Send the main script as a note. +list (APPEND CTEST_NOTES_FILES + "${CTEST_SCRIPT_DIRECTORY}/${CTEST_SCRIPT_NAME}" + "${CMAKE_CURRENT_LIST_FILE}" + "${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake" +) + +#----------------------------------------------------------------------------- +# Check for required variables. +# -------------------------- +foreach (req + CTEST_CMAKE_GENERATOR + CTEST_SITE + CTEST_BUILD_NAME + ) + if (NOT DEFINED ${req}) + message (FATAL_ERROR "The containing script must set ${req}") + endif () +endforeach () + +#----------------------------------------------------------------------------- +# Initialize the CTEST commands +#------------------------------ +if (CMAKE_GENERATOR_TOOLSET) + set (CTEST_CONFIGURE_TOOLSET "\"-T${CMAKE_GENERATOR_TOOLSET}\"") +else () + set (CTEST_CONFIGURE_TOOLSET) +endif() +if (CMAKE_GENERATOR_ARCHITECTURE) + set (CTEST_CONFIGURE_ARCHITECTURE "\"-A${CMAKE_GENERATOR_ARCHITECTURE}\"") +else () + set (CTEST_CONFIGURE_ARCHITECTURE) +endif() +set (CTEST_CONFIGURE_COMMAND + "${CTEST_CMAKE_COMMAND} -C \"${CTEST_SOURCE_DIRECTORY}/config/cmake/cacheinit.cmake\" -DCMAKE_BUILD_TYPE:STRING=${CTEST_CONFIGURATION_TYPE} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" ${CTEST_CONFIGURE_ARCHITECTURE} ${CTEST_CONFIGURE_TOOLSET} \"${CTEST_SOURCE_DIRECTORY}\"" +) +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +## -- set output to english +set (ENV{LC_MESSAGES} "en_EN") + +# Print summary information. +foreach (v + CTEST_SITE + CTEST_BUILD_NAME + CTEST_SOURCE_DIRECTORY + CTEST_BINARY_DIRECTORY + CTEST_CMAKE_GENERATOR + CTEST_CONFIGURATION_TYPE + CTEST_GIT_COMMAND + CTEST_CHECKOUT_COMMAND + CTEST_CONFIGURE_COMMAND + CTEST_SCRIPT_DIRECTORY + CTEST_USE_LAUNCHERS + ) + set (vars "${vars} ${v}=[${${v}}]\n") +endforeach () +message (STATUS "Dashboard script configuration:\n${vars}\n") + +#----------------------------------------------------------------------------- + +################################################################### +######### Following is for submission to CDash ############ +################################################################### +if (NOT DEFINED MODEL) + set (MODEL "Experimental") +endif () + +set (ENV{CI_SITE_NAME} ${CTEST_SITE}) +set (ENV{CI_BUILD_NAME} ${CTEST_BUILD_NAME}) +set (ENV{CI_MODEL} ${MODEL}) + +#----------------------------------------------------------------------------- + ## NORMAL process + ## -- LOCAL_UPDATE updates the source folder from svn + ## -- LOCAL_SUBMIT reports to CDash server + ## -- LOCAL_SKIP_TEST skips the test process (only builds) + ## -------------------------- + ctest_start (${MODEL} GROUP ${MODEL}) + if (LOCAL_UPDATE) + ctest_update (SOURCE "${CTEST_SOURCE_DIRECTORY}") + endif () + configure_file (${CTEST_SOURCE_DIRECTORY}/config/cmake/CTestCustom.cmake ${CTEST_BINARY_DIRECTORY}/CTestCustom.cmake) + ctest_read_custom_files ("${CTEST_BINARY_DIRECTORY}") + ctest_configure (BUILD "${CTEST_BINARY_DIRECTORY}" RETURN_VALUE res) + if (LOCAL_SUBMIT) + ctest_submit (PARTS Update Configure Notes) + endif () + if (${res} LESS 0 OR ${res} GREATER 0) + file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Configure: ${res}\n") + endif () + + ctest_build (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND RETURN_VALUE res NUMBER_ERRORS errval) + if (LOCAL_SUBMIT) + ctest_submit (PARTS Build) + endif () + if (${res} LESS 0 OR ${res} GREATER 0 OR ${errval} GREATER 0) + file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed ${errval} Build: ${res}\n") + endif () + + if (NOT LOCAL_SKIP_TEST) + ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res) + if (LOCAL_SUBMIT) + ctest_submit (PARTS Test) + endif () + if (${res} LESS 0 OR ${res} GREATER 0) + file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n") + endif () + endif () + if (LOCAL_SUBMIT) + ctest_submit (PARTS Done) + endif () +#----------------------------------------------------------------------------- diff --git a/SZ3/config/cmake/ConfigureChecks.cmake b/SZ3/config/cmake/ConfigureChecks.cmake new file mode 100644 index 00000000..c22af9c7 --- /dev/null +++ b/SZ3/config/cmake/ConfigureChecks.cmake @@ -0,0 +1,346 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +#----------------------------------------------------------------------------- +# Include all the necessary files for macros +#----------------------------------------------------------------------------- +include (CheckFunctionExists) +include (CheckIncludeFile) +include (CheckIncludeFiles) +include (CheckLibraryExists) +include (CheckSymbolExists) +include (CheckTypeSize) +include (CheckVariableExists) +include (TestBigEndian) + +# Check for Darwin (not just Apple - we also want to catch OpenDarwin) +if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set (HAVE_DARWIN 1) +endif () + +# Check for Solaris +if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + set (HAVE_SOLARIS 1) +endif () + +#----------------------------------------------------------------------------- +# This MACRO checks IF the symbol exists in the library and IF it +# does, it appends library to the list. +#----------------------------------------------------------------------------- +set (H5PL_LINK_LIBS "") +macro (CHECK_LIBRARY_EXISTS_CONCAT LIBRARY SYMBOL VARIABLE) + CHECK_LIBRARY_EXISTS ("${LIBRARY};${H5PL_LINK_LIBS}" ${SYMBOL} "" ${VARIABLE}) + if (${VARIABLE}) + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${LIBRARY}) + endif () +endmacro () + +# ---------------------------------------------------------------------- +# WINDOWS Hard code Values +# ---------------------------------------------------------------------- +set (WINDOWS) + +if (MINGW) + set (HAVE_MINGW 1) + set (WINDOWS 1) # MinGW tries to imitate Windows + set (CMAKE_REQUIRED_FLAGS "-DWIN32_LEAN_AND_MEAN=1 -DNOGDI=1") + set (HAVE_WINSOCK2_H 1) +endif () + +if (WIN32 AND NOT MINGW) + if (NOT UNIX) + set (WINDOWS 1) + set (CMAKE_REQUIRED_FLAGS "/DWIN32_LEAN_AND_MEAN=1 /DNOGDI=1") + if (MSVC) + set (HAVE_VISUAL_STUDIO 1) + endif () + endif () +endif () + +if (WINDOWS) + set (HDF_REQUIRED_LIBRARIES "ws2_32.lib;wsock32.lib") + set (HAVE_WIN32_API 1) + set (HAVE_LIBM 1) + set (HAVE_STRDUP 1) + set (HAVE_SYSTEM 1) + set (HAVE_LONGJMP 1) + if (NOT MINGW) + set (HAVE_GETHOSTNAME 1) + set (HAVE_FUNCTION 1) + endif () + if (NOT UNIX AND NOT CYGWIN) + set (HAVE_LIBWS2_32 1) + set (HAVE_LIBWSOCK32 1) + endif () +endif () + +# ---------------------------------------------------------------------- +# END of WINDOWS Hard code Values +# ---------------------------------------------------------------------- + +if (NOT WINDOWS) + TEST_BIG_ENDIAN (WORDS_BIGENDIAN) +endif () + +#----------------------------------------------------------------------------- +# Check IF header file exists and add it to the list. +#----------------------------------------------------------------------------- +macro (CHECK_INCLUDE_FILE_CONCAT FILE VARIABLE) + CHECK_INCLUDE_FILES ("${USE_INCLUDES};${FILE}" ${VARIABLE}) + if (${VARIABLE}) + set (USE_INCLUDES ${USE_INCLUDES} ${FILE}) + endif () +endmacro () + +#----------------------------------------------------------------------------- +# Check for the existence of certain header files +#----------------------------------------------------------------------------- +CHECK_INCLUDE_FILE_CONCAT ("sys/stat.h" HAVE_SYS_STAT_H) +CHECK_INCLUDE_FILE_CONCAT ("sys/types.h" HAVE_SYS_TYPES_H) +CHECK_INCLUDE_FILE_CONCAT ("stddef.h" HAVE_STDDEF_H) +CHECK_INCLUDE_FILE_CONCAT ("stdint.h" HAVE_STDINT_H) +CHECK_INCLUDE_FILE_CONCAT ("unistd.h" HAVE_UNISTD_H) + +# Windows +CHECK_INCLUDE_FILE_CONCAT ("io.h" HAVE_IO_H) +if (NOT CYGWIN) + CHECK_INCLUDE_FILE_CONCAT ("winsock2.h" HAVE_WINSOCK_H) +endif () + +CHECK_INCLUDE_FILE_CONCAT ("pthread.h" HAVE_PTHREAD_H) +CHECK_INCLUDE_FILE_CONCAT ("string.h" HAVE_STRING_H) +CHECK_INCLUDE_FILE_CONCAT ("strings.h" HAVE_STRINGS_H) +CHECK_INCLUDE_FILE_CONCAT ("stdlib.h" HAVE_STDLIB_H) +CHECK_INCLUDE_FILE_CONCAT ("memory.h" HAVE_MEMORY_H) +CHECK_INCLUDE_FILE_CONCAT ("dlfcn.h" HAVE_DLFCN_H) +CHECK_INCLUDE_FILE_CONCAT ("fcntl.h" HAVE_FCNTL_H) +CHECK_INCLUDE_FILE_CONCAT ("inttypes.h" HAVE_INTTYPES_H) + +# _Bool type support +CHECK_INCLUDE_FILE_CONCAT (stdbool.h HAVE_STDBOOL_H) + +#----------------------------------------------------------------------------- +# Check for the math library "m" +#----------------------------------------------------------------------------- +if (MINGW OR NOT WINDOWS) + CHECK_LIBRARY_EXISTS_CONCAT ("m" ceil HAVE_LIBM) + CHECK_LIBRARY_EXISTS_CONCAT ("dl" dlopen HAVE_LIBDL) + CHECK_LIBRARY_EXISTS_CONCAT ("ws2_32" WSAStartup HAVE_LIBWS2_32) + CHECK_LIBRARY_EXISTS_CONCAT ("wsock32" gethostbyname HAVE_LIBWSOCK32) +endif () + +# UCB (BSD) compatibility library +CHECK_LIBRARY_EXISTS_CONCAT ("ucb" gethostname HAVE_LIBUCB) + +# For other tests to use the same libraries +set (HDF_REQUIRED_LIBRARIES ${HDF_REQUIRED_LIBRARIES} ${H5PL_LINK_LIBS}) + +set (USE_INCLUDES "") +if (WINDOWS) + set (USE_INCLUDES ${USE_INCLUDES} "windows.h") +endif () + +# For other specific tests, use this MACRO. +macro (HDF_FUNCTION_TEST OTHER_TEST) + if (NOT DEFINED ${OTHER_TEST}) + set (MACRO_CHECK_FUNCTION_DEFINITIONS "-D${OTHER_TEST} ${CMAKE_REQUIRED_FLAGS}") + + foreach (def + HAVE_UNISTD_H + HAVE_SYS_TYPES_H + ) + if ("${${def}}") + set (MACRO_CHECK_FUNCTION_DEFINITIONS "${MACRO_CHECK_FUNCTION_DEFINITIONS} -D${def}") + endif () + endforeach () + + if (LARGEFILE) + set (MACRO_CHECK_FUNCTION_DEFINITIONS + "${MACRO_CHECK_FUNCTION_DEFINITIONS} -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_LARGEFILE_SOURCE" + ) + endif () + + message (TRACE "Performing ${OTHER_TEST}") + try_compile (${OTHER_TEST} + ${CMAKE_BINARY_DIR} + ${H5SZ3_RESOURCES_DIR}/H5PLTests.c + COMPILE_DEFINITIONS "${MACRO_CHECK_FUNCTION_DEFINITIONS}" + LINK_LIBRARIES "${HDF_REQUIRED_LIBRARIES}" + OUTPUT_VARIABLE OUTPUT + ) + if (${OTHER_TEST}) + set (${OTHER_TEST} 1 CACHE INTERNAL "Other test ${FUNCTION}") + message (VERBOSE "Performing Other Test ${OTHER_TEST} - Success") + else () + message (VERBOSE "Performing Other Test ${OTHER_TEST} - Failed") + set (${OTHER_TEST} "" CACHE INTERNAL "Other test ${FUNCTION}") + file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Performing Other Test ${OTHER_TEST} failed with the following output:\n" + "${OUTPUT}\n" + ) + endif () + endif () +endmacro () + +#----------------------------------------------------------------------------- +# Platform-specific flags +#----------------------------------------------------------------------------- + +set (HDF_EXTRA_C_FLAGS) + +# Linux-specific flags +if (CMAKE_SYSTEM_NAME MATCHES "Linux") + # This was originally defined as _POSIX_SOURCE which was updated to + # _POSIX_C_SOURCE=199506L to expose a greater amount of POSIX + # functionality so clock_gettime and CLOCK_MONOTONIC are defined + # correctly. This was later updated to 200112L so that + # posix_memalign() is visible for the direct VFD code on Linux + # systems. + # POSIX feature information can be found in the gcc manual at: + # http://www.gnu.org/s/libc/manual/html_node/Feature-Test-Macros.html + set (HDF_EXTRA_C_FLAGS -D_POSIX_C_SOURCE=200809L) + + # Set up large file support. This is only necessary on 32-bit systems + # but is used on all Linux systems. It has no effect on 64-bit systems + # so it's not worth hacking up a 32/64-bit test to selectively include it. + # + # The library currently does not use any of the 64-flavored API calls + # or types + set (HDF_EXTRA_C_FLAGS ${HDF_EXTRA_C_FLAGS} -D_LARGEFILE_SOURCE) + set (HDF_EXTRA_C_FLAGS ${HDF_EXTRA_C_FLAGS} -D_FILE_OFFSET_BITS=64) + + set (CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} ${HDF_EXTRA_C_FLAGS}) +endif () + +# As of 2024, both AIX and Solaris are uncommon, but still exist! The default +# compiler options are also often set to -m32, which produces 32-bit binaries. + +# 32-bit AIX compiles might require _LARGE_FILES, but we don't have a system on +# which to test this (yet). +# +# https://www.ibm.com/docs/en/aix/7.1?topic=volumes-writing-programs-that-access-large-files + +# 32-bit Solaris probably needs _LARGEFILE_SOURCE and _FILE_OFFSET_BITS=64, +# as in Linux, above. +# +# https://docs.oracle.com/cd/E23824_01/html/821-1474/lfcompile-5.html + +# MinGW and Cygwin +if (MINGW OR CYGWIN) + set (CMAKE_REQUIRED_DEFINITIONS + "${CURRENT_TEST_DEFINITIONS} -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE" + ) +endif () + +#----------------------------------------------------------------------------- +# Check the size in bytes of all the int and float types +#----------------------------------------------------------------------------- +macro (HDF_CHECK_TYPE_SIZE type var) + set (aType ${type}) + set (aVar ${var}) + message (TRACE "Checking size of ${aType} and storing into ${aVar}") + CHECK_TYPE_SIZE (${aType} ${aVar}) + if (NOT ${aVar}) + set (${aVar} 0 CACHE INTERNAL "SizeOf for ${aType}") + message (TRACE "Size of ${aType} was NOT Found") + endif () +endmacro () + +HDF_CHECK_TYPE_SIZE (char SIZEOF_CHAR) +HDF_CHECK_TYPE_SIZE (short SIZEOF_SHORT) +HDF_CHECK_TYPE_SIZE (int SIZEOF_INT) +HDF_CHECK_TYPE_SIZE (unsigned SIZEOF_UNSIGNED) +if (NOT APPLE) + HDF_CHECK_TYPE_SIZE (long SIZEOF_LONG) +endif () +HDF_CHECK_TYPE_SIZE ("long long" SIZEOF_LONG_LONG) +HDF_CHECK_TYPE_SIZE (__int64 SIZEOF___INT64) +if (NOT SIZEOF___INT64) + set (SIZEOF___INT64 0) +endif () + +HDF_CHECK_TYPE_SIZE (float SIZEOF_FLOAT) +HDF_CHECK_TYPE_SIZE (double SIZEOF_DOUBLE) +HDF_CHECK_TYPE_SIZE ("long double" SIZEOF_LONG_DOUBLE) + +HDF_CHECK_TYPE_SIZE (int8_t SIZEOF_INT8_T) +HDF_CHECK_TYPE_SIZE (uint8_t SIZEOF_UINT8_T) +HDF_CHECK_TYPE_SIZE (int_least8_t SIZEOF_INT_LEAST8_T) +HDF_CHECK_TYPE_SIZE (uint_least8_t SIZEOF_UINT_LEAST8_T) +HDF_CHECK_TYPE_SIZE (int_fast8_t SIZEOF_INT_FAST8_T) +HDF_CHECK_TYPE_SIZE (uint_fast8_t SIZEOF_UINT_FAST8_T) + +HDF_CHECK_TYPE_SIZE (int16_t SIZEOF_INT16_T) +HDF_CHECK_TYPE_SIZE (uint16_t SIZEOF_UINT16_T) +HDF_CHECK_TYPE_SIZE (int_least16_t SIZEOF_INT_LEAST16_T) +HDF_CHECK_TYPE_SIZE (uint_least16_t SIZEOF_UINT_LEAST16_T) +HDF_CHECK_TYPE_SIZE (int_fast16_t SIZEOF_INT_FAST16_T) +HDF_CHECK_TYPE_SIZE (uint_fast16_t SIZEOF_UINT_FAST16_T) + +HDF_CHECK_TYPE_SIZE (int32_t SIZEOF_INT32_T) +HDF_CHECK_TYPE_SIZE (uint32_t SIZEOF_UINT32_T) +HDF_CHECK_TYPE_SIZE (int_least32_t SIZEOF_INT_LEAST32_T) +HDF_CHECK_TYPE_SIZE (uint_least32_t SIZEOF_UINT_LEAST32_T) +HDF_CHECK_TYPE_SIZE (int_fast32_t SIZEOF_INT_FAST32_T) +HDF_CHECK_TYPE_SIZE (uint_fast32_t SIZEOF_UINT_FAST32_T) + +HDF_CHECK_TYPE_SIZE (int64_t SIZEOF_INT64_T) +HDF_CHECK_TYPE_SIZE (uint64_t SIZEOF_UINT64_T) +HDF_CHECK_TYPE_SIZE (int_least64_t SIZEOF_INT_LEAST64_T) +HDF_CHECK_TYPE_SIZE (uint_least64_t SIZEOF_UINT_LEAST64_T) +HDF_CHECK_TYPE_SIZE (int_fast64_t SIZEOF_INT_FAST64_T) +HDF_CHECK_TYPE_SIZE (uint_fast64_t SIZEOF_UINT_FAST64_T) + +if (NOT APPLE) + HDF_CHECK_TYPE_SIZE (size_t SIZEOF_SIZE_T) + HDF_CHECK_TYPE_SIZE (ssize_t SIZEOF_SSIZE_T) + if (NOT SIZEOF_SSIZE_T) + set (SIZEOF_SSIZE_T 0) + endif () + if (MINGW OR NOT WINDOWS) + HDF_CHECK_TYPE_SIZE (ptrdiff_t SIZEOF_PTRDIFF_T) + endif () +endif () + +HDF_CHECK_TYPE_SIZE (off_t SIZEOF_OFF_T) +HDF_CHECK_TYPE_SIZE (off64_t SIZEOF_OFF64_T) +if (NOT SIZEOF_OFF64_T) + set (SIZEOF_OFF64_T 0) +endif () +HDF_CHECK_TYPE_SIZE (time_t SIZEOF_TIME_T) + +#----------------------------------------------------------------------------- +# Extra C99 types +#----------------------------------------------------------------------------- + +# _Bool type support +if (HAVE_STDBOOL_H) + set (CMAKE_EXTRA_INCLUDE_FILES stdbool.h) + HDF_CHECK_TYPE_SIZE (bool SIZEOF_BOOL) +else () + HDF_CHECK_TYPE_SIZE (_Bool SIZEOF_BOOL) +endif () + +#----------------------------------------------------------------------------- +# Check for some functions that are used +# +CHECK_FUNCTION_EXISTS (vprintf HAVE_VPRINTF) +CHECK_FUNCTION_EXISTS (_doprnt HAVE_DOPRNT) +CHECK_FUNCTION_EXISTS (memset HAVE_MEMSET) + +if (MINGW OR NOT WINDOWS) + foreach (test + HAVE_ATTRIBUTE + SYSTEM_SCOPE_THREADS + ) + HDF_FUNCTION_TEST (${test}) + endforeach () +endif () diff --git a/SZ3/config/cmake/H5PLTests.c b/SZ3/config/cmake/H5PLTests.c new file mode 100644 index 00000000..aa537df6 --- /dev/null +++ b/SZ3/config/cmake/H5PLTests.c @@ -0,0 +1,176 @@ +#define SIMPLE_TEST(x) \ + int main() \ + { \ + x; \ + return 0; \ + } + +#ifdef VSNPRINTF_WORKS +#include +#include +#include + +int +test_vsnprintf(const char *fmt, ...) +{ + va_list ap; + char *s = malloc(16); + int ret; + + va_start(ap, fmt); + ret = vsnprintf(s, 16, "%s", ap); + va_end(ap); + + return (ret != 42 ? 1 : 0); +} + +int +main(void) +{ + return (test_vsnprintf("%s", "A string that is longer than 16 characters")); +} +#endif + +#ifdef STDC_HEADERS +#include +#include +#include +#include +int +main() +{ + return 0; +} +#endif /* STDC_HEADERS */ + +#ifdef HAVE_ATTRIBUTE + +#if 0 +static void test int __attribute((unused)) var) +{ + int __attribute__((unused)) x = var; +} + +int main(void) +{ + test(19); +} + +#else +int +main() +{ + int __attribute__((unused)) x; + return 0; +} +#endif + +#endif /* HAVE_ATTRIBUTE */ + +#ifdef SYSTEM_SCOPE_THREADS +#include +#include + +int +main(void) +{ + pthread_attr_t attribute; + int ret; + + pthread_attr_init(&attribute); + ret = pthread_attr_setscope(&attribute, PTHREAD_SCOPE_SYSTEM); + if (ret == 0) + return 0; + return 1; +} + +#endif /* SYSTEM_SCOPE_THREADS */ + +#ifdef HAVE_OFF64_T +#include +int +main() +{ + off64_t n = 0; + return (int)n; +} +#endif + +#ifdef HAVE_STAT64_STRUCT +#include +#include ], +struct stat64 sb; +int +main() +{ + return 0; +} +#endif + +#ifdef TEST_LFS_WORKS +/* Return 0 when LFS is available and 1 otherwise. */ +#define _LARGEFILE_SOURCE +#define _LARGEFILE64_SOURCE +#define _LARGE_FILES +#define _FILE_OFFSET_BITS 64 +#include +#include +#include +#include + +int +main(int argc, char **argv) +{ + /* check that off_t can hold 2^63 - 1 and perform basic operations... */ +#define OFF_T_64 (((off_t)1 << 62) - 1 + ((off_t)1 << 62)) + if (OFF_T_64 % 2147483647 != 1) + return 1; + + /* stat breaks on SCO OpenServer */ + struct stat buf; + stat(argv[0], &buf); + if (!S_ISREG(buf.st_mode)) + return 2; + + FILE *file = fopen(argv[0], "r"); + off_t offset = ftello(file); + fseek(file, offset, SEEK_CUR); + fclose(file); + return 0; +} +#endif + +#ifdef CXX_HAVE_OFFSETOF + +#include +#include + +#ifdef FC_DUMMY_MAIN +#ifndef FC_DUMMY_MAIN_EQ_F77 +#ifdef __cplusplus +extern "C" +#endif + int + FC_DUMMY_MAIN() +{ + return 1; +} +#endif +#endif +int +main() +{ + + struct index_st { + unsigned char type; + unsigned char num; + unsigned int len; + }; + typedef struct index_st index_t; + int x, y; + x = offsetof(struct index_st, len); + y = offsetof(index_t, num); + return 0; +} + +#endif diff --git a/SZ3/config/cmake/H5PL_Examples.cmake.in b/SZ3/config/cmake/H5PL_Examples.cmake.in new file mode 100644 index 00000000..87df21d0 --- /dev/null +++ b/SZ3/config/cmake/H5PL_Examples.cmake.in @@ -0,0 +1,128 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +cmake_minimum_required(VERSION 3.18) +############################################################################################################### +# This script will build and run the examples from a folder +# Execute from a command line: +# ctest -S H5PL_Examples.cmake,OPTION=VALUE -C Release -VV -O test.log +############################################################################################################### +# Minimum where plugins are installed within hdf5 library install, /usr/local/HDF_Group/HDF5/@H5PL_PACKAGE_VERSION@/lib/plugin, +# then is /usr/local/HDF_Group/HDF5/@H5PL_PACKAGE_VERSION@ +# ctest -S H5PL_Examples.cmake,HDF5INSTALLDIR= -C Release -VV -O test.log +############################################################################################################### + +set (CTEST_CMAKE_GENERATOR "@CMAKE_GENERATOR@") +if("@CMAKE_GENERATOR_TOOLSET@") + set (CMAKE_GENERATOR_TOOLSET "@CMAKE_GENERATOR_TOOLSET@") +endif() +if("@CMAKE_GENERATOR_ARCHITECTURE@") + set (CMAKE_GENERATOR_ARCHITECTURE "@CMAKE_GENERATOR_ARCHITECTURE@") +endif() +set (CTEST_DASHBOARD_ROOT ${CTEST_SCRIPT_DIRECTORY}) + +# handle input parameters to script. +#HDF5INSTALLDIR - HDF5 library root folder +#INSTALLDIR - HDF5 plugin root folder +#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo +#CTEST_SOURCE_NAME - name of source folder; HDFPLExamples +#STATIC_ONLY - Default is YES +if(DEFINED CTEST_SCRIPT_ARG) + # transform ctest script arguments of the form + # script.ctest,var1=value1,var2=value2 + # to variables with the respective names set to the respective values + string(REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}") + foreach(current_var ${script_args}) + if("${current_var}" MATCHES "^([^=]+)=(.+)$") + set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}") + endif() + endforeach() +endif() + +################################################################### +### Following Line is one of [Release, RelWithDebInfo, Debug] ##### +set (CTEST_CONFIGURATION_TYPE "$ENV{CMAKE_CONFIG_TYPE}") +if(NOT DEFINED CTEST_CONFIGURATION_TYPE) + set (CTEST_CONFIGURATION_TYPE "Release") +endif() +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCTEST_CONFIGURATION_TYPE:STRING=${CTEST_CONFIGURATION_TYPE}") +################################################################## + +if(NOT DEFINED HDF5INSTALLDIR) + set (HDF5INSTALLDIR "${CTEST_DASHBOARD_ROOT}/HDF_Group/HDF5/@H5PL_PACKAGE_VERSION@") +endif() +if(NOT DEFINED INSTALLDIR) + set (INSTALLDIR "${HDF5INSTALLDIR}") +endif() + +if(NOT DEFINED CTEST_SOURCE_NAME) + set (CTEST_SOURCE_NAME "HDFPLExamples") +endif() + +if(NOT DEFINED STATIC_ONLY) + set (STATICLIBRARIES "YES") +else(NOT DEFINED STATIC_ONLY) + set (STATICLIBRARIES "NO") +endif() + +if(NOT DEFINED CTEST_SITE) + set (CTEST_SITE "local") +endif() +if(NOT DEFINED CTEST_BUILD_NAME) + set (CTEST_BUILD_NAME "examples") +endif() +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDNAME:STRING=${CTEST_BUILD_NAME}") + +#TAR_SOURCE - name of tarfile +#if(NOT DEFINED TAR_SOURCE) +# set (CTEST_USE_TAR_SOURCE "HDFPLExamples-@H5PL_PACKAGE_VERSION@-Source") +#endif() + +############################################################################################################### +if(WIN32) + set (SITE_OS_NAME "Windows") + set (ENV{HDF5_ROOT} "${HDF5INSTALLDIR}/") + set (CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}\\build) + set (CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_SOURCE_NAME}") + set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_BINARY_NAME}") + set (ENV{PATH} "$ENV{PATH};${HDF5INSTALLDIR}") + set (HDF5_PLUGIN_ROOT "${INSTALLDIR}/lib/plugin") +else() + set (ENV{HDF5_ROOT} "${HDF5INSTALLDIR}/") + set (ENV{LD_LIBRARY_PATH} "${HDF5INSTALLDIR}/lib") + set (CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}/build) + set (CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_SOURCE_NAME}") + set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_BINARY_NAME}") + set (ENV{PATH} "$ENV{PATH}:${HDF5INSTALLDIR}") + set (HDF5_PLUGIN_ROOT "${INSTALLDIR}/lib/plugin") +endif() +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACKAGE_NAME:STRING=@HDF5_PACKAGE_NAME@") +set (ENV{HDF5_PLUGIN_PATH} "${HDF5_PLUGIN_ROOT}") + +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DDISABLE_H5PL_ENCODER:BOOL=@DISABLE_H5SZ3_ENCODER@") +### use a toolchain file (supported everywhere) #### +if("@CMAKE_TOOLCHAIN_FILE@") +# set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=@CMAKE_TOOLCHAIN_FILE@") +endif() + +############################################################################################################### +# For any comments please contact cdashhelp@hdfgroup.org +# +############################################################################################################### +if(WIN32) + include(${CTEST_DASHBOARD_ROOT}\\HDFPLoptions.cmake) + include(${CTEST_DASHBOARD_ROOT}\\CTestScript.cmake) +else() + include(${CTEST_DASHBOARD_ROOT}/HDFPLoptions.cmake) + include(${CTEST_DASHBOARD_ROOT}/CTestScript.cmake) +endif() +############################################################################################################## +message(STATUS "DONE") \ No newline at end of file diff --git a/SZ3/config/cmake/HDFMacros.cmake b/SZ3/config/cmake/HDFMacros.cmake new file mode 100644 index 00000000..60e69e7d --- /dev/null +++ b/SZ3/config/cmake/HDFMacros.cmake @@ -0,0 +1,519 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +#------------------------------------------------------------------------------- +# Setup output Directories +#----------------------------------------------------------------------------- +macro (SET_HDF_OUTPUT_DIRS package_prefix) + if (NOT ${package_prefix}_EXTERNALLY_CONFIGURED) + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." + ) + set (CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" + ) + set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." + ) + set (CMAKE_Fortran_MODULE_DIRECTORY + ${PROJECT_BINARY_DIR}/mod CACHE PATH "Single Directory for all fortran modules." + ) + get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if(_isMultiConfig) + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}) + set (CMAKE_PDB_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all pdb files." + ) + else () + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + endif () + else () + # if we are externally configured, but the project uses old cmake scripts + # this may not be set and utilities like H5detect will fail + if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (SET_HDF_BUILD_TYPE) + get_property (_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if (_isMultiConfig) + # HDF_CFG_BUILD_TYPE is used in the Fortran install commands for the build location of the .mod files + set (HDF_CFG_BUILD_TYPE \${CMAKE_INSTALL_CONFIG_NAME}) + if (CMAKE_BUILD_TYPE) + # set the default to the specified command line define + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + # set the default to the MultiConfig variable + set (HDF_CFG_NAME "$") + endif () + else () + set (HDF_CFG_BUILD_TYPE ".") + if (CMAKE_BUILD_TYPE) + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + set (HDF_CFG_NAME "Release") + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (SET_GLOBAL_VARIABLE name value) + set (${name} ${value} CACHE INTERNAL "Used to pass variables between directories" FORCE) +endmacro () + +#------------------------------------------------------------------------------- +macro (IDE_GENERATED_PROPERTIES SOURCE_PATH HEADERS SOURCES) + #set (source_group_path "Source/AIM/${NAME}") + string (REPLACE "/" "\\\\" source_group_path ${SOURCE_PATH}) + source_group (${source_group_path} FILES ${HEADERS} ${SOURCES}) + + #-- The following is needed if we ever start to use OS X Frameworks but only + #-- works on CMake 2.6 and greater + #set_property (SOURCE ${HEADERS} + # PROPERTY MACOSX_PACKAGE_LOCATION Headers/${NAME} + #) +endmacro () + +#------------------------------------------------------------------------------- +macro (IDE_SOURCE_PROPERTIES SOURCE_PATH HEADERS SOURCES) + # install (FILES ${HEADERS} + # DESTINATION include/R3D/${NAME} + # COMPONENT Headers + # ) + + string (REPLACE "/" "\\\\" source_group_path ${SOURCE_PATH} ) + source_group (${source_group_path} FILES ${HEADERS} ${SOURCES}) + + #-- The following is needed if we ever start to use OS X Frameworks but only + #-- works on CMake 2.6 and greater + #set_property (SOURCE ${HEADERS} + # PROPERTY MACOSX_PACKAGE_LOCATION Headers/${NAME} + #) +endmacro () + +#------------------------------------------------------------------------------- +macro (INSTALL_TARGET_PDB libtarget targetdestination targetcomponent) + option (DISABLE_PDB_FILES "Do not install PDB files" OFF) + mark_as_advanced (DISABLE_PDB_FILES) + if (WIN32 AND MSVC AND NOT DISABLE_PDB_FILES) + get_target_property (target_type ${libtarget} TYPE) + if (${target_type} MATCHES "SHARED") + set (targetfilename $) + else () + get_property (target_name TARGET ${libtarget} PROPERTY "$,$>,OUTPUT_NAME_DEBUG,OUTPUT_NAME_RELWITHDEBINFO>") + set (targetfilename $/${target_name}.pdb) + endif () + install ( + FILES ${targetfilename} + DESTINATION ${targetdestination} + CONFIGURATIONS Debug RelWithDebInfo + COMPONENT ${targetcomponent} + OPTIONAL + ) + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (INSTALL_PROGRAM_PDB progtarget targetdestination targetcomponent) + if (WIN32 AND MSVC) + install ( + FILES $ + DESTINATION ${targetdestination} + CONFIGURATIONS Debug RelWithDebInfo + COMPONENT ${targetcomponent} + OPTIONAL + ) + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF_SET_BASE_OPTIONS libtarget libname libtype) + if (${libtype} MATCHES "SHARED") + set (LIB_RELEASE_NAME "${libname}") + set (LIB_DEBUG_NAME "${libname}${CMAKE_DEBUG_POSTFIX}") + else () + if (WIN32 AND NOT MINGW) + if ("${ARGN}" STREQUAL "NOPREFIX") + set (LIB_RELEASE_NAME "${libname}") + set (LIB_DEBUG_NAME "${libname}${CMAKE_DEBUG_POSTFIX}") + else () + set (LIB_RELEASE_NAME "lib${libname}") + set (LIB_DEBUG_NAME "lib${libname}${CMAKE_DEBUG_POSTFIX}") + endif () + else () + set (LIB_RELEASE_NAME "${libname}") + set (LIB_DEBUG_NAME "${libname}${CMAKE_DEBUG_POSTFIX}") + endif () + endif () + + set_target_properties (${libtarget} PROPERTIES + OUTPUT_NAME ${LIB_RELEASE_NAME} +# OUTPUT_NAME_DEBUG ${LIB_DEBUG_NAME} + OUTPUT_NAME_RELEASE ${LIB_RELEASE_NAME} + OUTPUT_NAME_MINSIZEREL ${LIB_RELEASE_NAME} + OUTPUT_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} + ) + + if (${libtype} MATCHES "STATIC") + if (WIN32) + set_target_properties (${libtarget} PROPERTIES + COMPILE_PDB_NAME_DEBUG ${LIB_DEBUG_NAME} + COMPILE_PDB_NAME_RELEASE ${LIB_RELEASE_NAME} + COMPILE_PDB_NAME_MINSIZEREL ${LIB_RELEASE_NAME} + COMPILE_PDB_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" + ) + endif () + endif () + + option (HDF_MSVC_NAMING_CONVENTION "Use MSVC Naming conventions for Shared Libraries" OFF) + if (HDF_MSVC_NAMING_CONVENTION AND MINGW AND ${libtype} MATCHES "SHARED") + set_target_properties (${libtarget} PROPERTIES + IMPORT_SUFFIX ".lib" + IMPORT_PREFIX "" + PREFIX "" + ) + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF_SET_LIB_VERSIONS pkg_name libtarget defaultlibname libtype) + set (libname "${defaultlibname}") + HDF_SET_BASE_OPTIONS (${libtarget} ${libname} ${libtype} ${ARGN}) + + if (${libtype} MATCHES "SHARED") + set (LIB_PACKAGE_SOVERSION ${${pkg_name}_VERS_MAJOR}) + if (WIN32) + set (LIB_VERSION ${${pkg_name}_PACKAGE_VERSION_MAJOR}) + else () + set (LIB_VERSION ${${pkg_name}_PACKAGE_VERSION}) + endif () + set_target_properties (${libtarget} PROPERTIES VERSION ${LIB_VERSION}) + if (WIN32) + set (${libname} "${libname}-${LIB_PACKAGE_SOVERSION}") + else () + set_target_properties (${libtarget} PROPERTIES SOVERSION ${LIB_PACKAGE_SOVERSION}) + endif () + endif () + + #-- Apple Specific install_name for libraries + if (APPLE) + option (${pkg_name}_BUILD_WITH_INSTALL_NAME "Build with library install_name set to the installation path" OFF) + if (${pkg_name}_BUILD_WITH_INSTALL_NAME) + set_target_properties(${libtarget} PROPERTIES + LINK_FLAGS "-current_version ${${pkg_name}_PACKAGE_VERSION} -compatibility_version ${${pkg_name}_PACKAGE_VERSION}" + INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib" + BUILD_WITH_INSTALL_RPATH ${${pkg_name}_BUILD_WITH_INSTALL_NAME} + ) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF_IMPORT_SET_LIB_OPTIONS libtarget libname libtype libversion) + HDF_SET_BASE_OPTIONS (${libtarget} ${libname} ${libtype} ${ARGN}) + + if (${importtype} MATCHES "IMPORT") + set (importprefix "${CMAKE_STATIC_LIBRARY_PREFIX}") + endif () + if (${HDF_CFG_NAME} MATCHES "Debug") + set (IMPORT_LIB_NAME ${LIB_DEBUG_NAME}) + else () + set (IMPORT_LIB_NAME ${LIB_RELEASE_NAME}) + endif () + + if (${libtype} MATCHES "SHARED") + if (WIN32) + if (MINGW) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}.lib" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + else () + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + endif () + else () + if (MINGW) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}.lib" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + elseif (CYGWIN) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + else () + set_target_properties (${libtarget} PROPERTIES + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + IMPORTED_SONAME "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${libversion}" + SOVERSION "${libversion}" + ) + endif () + endif () + else () + if (WIN32 AND NOT MINGW) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${IMPORT_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + ) + else () + set_target_properties (${libtarget} PROPERTIES + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_STATIC_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + ) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (TARGET_C_PROPERTIES wintarget libtype) + target_compile_options(${wintarget} PRIVATE + "$<$:${WIN_COMPILE_FLAGS}>" + "$<$:${WIN_COMPILE_FLAGS}>" + ) + if(MSVC) + set_property(TARGET ${wintarget} APPEND PROPERTY LINK_FLAGS "${WIN_LINK_FLAGS}") + endif() +endmacro () + +#----------------------------------------------------------------------------- +# Configure the README.txt file for the binary package +#----------------------------------------------------------------------------- +macro (PLUGIN_README_PROPERTIES pkg_name) + set (BINARY_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) + set (BINARY_PLATFORM "${CMAKE_SYSTEM_NAME}") + if (WIN32) + set (BINARY_EXAMPLE_ENDING "zip") + set (BINARY_INSTALL_ENDING "msi") + set (BINARY_COMPRESS_ENDING "zip") + if (CMAKE_CL_64) + set (BINARY_SYSTEM_NAME "win64") + else () + set (BINARY_SYSTEM_NAME "win32") + endif () + if (${CMAKE_SYSTEM_VERSION} MATCHES "6.1") + set (BINARY_PLATFORM "${BINARY_PLATFORM} 7") + elseif (${CMAKE_SYSTEM_VERSION} MATCHES "6.2") + set (BINARY_PLATFORM "${BINARY_PLATFORM} 8") + elseif (${CMAKE_SYSTEM_VERSION} MATCHES "6.3") + set (BINARY_PLATFORM "${BINARY_PLATFORM} 10") + endif () + if (CMAKE_C_COMPILER_ID MATCHES "Intel") + set (BINARY_PLATFORM "${BINARY_PLATFORM} Intel") + if (${CMAKE_C_COMPILER_VERSION} MATCHES "^17.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel 17") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^18.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel 18") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel 19") + else () + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel ${CMAKE_C_COMPILER_VERSION}") + endif () + elseif (CMAKE_C_COMPILER_ID MATCHES "MSVC") + set (BINARY_PLATFORM "${BINARY_PLATFORM} ${MSVC_C_ARCHITECTURE_ID}") + if (${CMAKE_C_COMPILER_VERSION} MATCHES "^16.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2010") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^15.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2008") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^17.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2012") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^18.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2013") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.*") + if (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.0.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2015") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.1.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2017") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.2.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2019") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.3.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2022") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.4.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2022") + else () + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO ????") + endif () + else () + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO ${CMAKE_C_COMPILER_VERSION}") + endif () + endif () + elseif (APPLE) + set (BINARY_EXAMPLE_ENDING "tar.gz") + set (BINARY_COMPRESS_ENDING "tar.gz") + set (BINARY_INSTALL_ENDING "sh") # if packaging changes - use dmg + set (BINARY_PLATFORM "${BINARY_PLATFORM} ${CMAKE_SYSTEM_VERSION} ${CMAKE_SYSTEM_PROCESSOR}") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using ${CMAKE_C_COMPILER_ID} C ${CMAKE_C_COMPILER_VERSION}") + else () + set (BINARY_EXAMPLE_ENDING "tar.gz") + set (BINARY_COMPRESS_ENDING "tar.gz") + set (BINARY_INSTALL_ENDING "sh/deb/rpm") + set (BINARY_PLATFORM "${BINARY_PLATFORM} ${CMAKE_SYSTEM_VERSION} ${CMAKE_SYSTEM_PROCESSOR}") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using ${CMAKE_C_COMPILER_ID} C ${CMAKE_C_COMPILER_VERSION}") + endif () + + configure_file ( + ${${pkg_name}_RESOURCES_DIR}/README.txt.cmake.in + ${CMAKE_BINARY_DIR}/README.txt @ONLY + ) +endmacro () + +macro (HDFTEST_COPY_FILE src dest target) + add_custom_command( + OUTPUT "${dest}" + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy_if_different "${src}" "${dest}" + DEPENDS "${src}" + ) + list (APPEND ${target}_list "${dest}") +endmacro () + +macro (HDF_DIR_PATHS package_prefix) + option (H5PL_USE_GNU_DIRS "ON to use GNU Coding Standard install directory variables, OFF to use historical settings" OFF) + if (H5PL_USE_GNU_DIRS) + include(GNUInstallDirs) + if (NOT ${package_prefix}_INSTALL_BIN_DIR) + set (${package_prefix}_INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_LIB_DIR) + set (${package_prefix}_INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_JAR_DIR) + set (${package_prefix}_INSTALL_JAR_DIR ${CMAKE_INSTALL_LIBDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_INCLUDE_DIR) + set (${package_prefix}_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_MODULE_DIR) + set (${package_prefix}_INSTALL_MODULE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/mod) + endif () + if (NOT ${package_prefix}_INSTALL_DATA_DIR) + set (${package_prefix}_INSTALL_DATA_DIR ${CMAKE_INSTALL_DATADIR}) + endif () + if (NOT ${package_prefix}_INSTALL_CMAKE_DIR) + set (${package_prefix}_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake) + endif () + if (NOT ${package_prefix}_INSTALL_DOC_DIR) + set (${package_prefix}_INSTALL_DOC_DIR ${CMAKE_INSTALL_DOCDIR}) + endif () + message(STATUS "GNU: ${${package_prefix}_INSTALL_DOC_DIR}") + endif () + + if (APPLE) + option (${package_prefix}_BUILD_FRAMEWORKS "ON to build as frameworks libraries, OFF to build according to BUILD_SHARED_LIBS" OFF) + endif () + + if (NOT ${package_prefix}_INSTALL_BIN_DIR) + set (${package_prefix}_INSTALL_BIN_DIR bin) + endif () + if (NOT ${package_prefix}_INSTALL_LIB_DIR) + if (APPLE) + if (${package_prefix}_BUILD_FRAMEWORKS) + set (${package_prefix}_INSTALL_JAR_DIR ../Java) + else () + set (${package_prefix}_INSTALL_JAR_DIR lib) + endif () + set (${package_prefix}_INSTALL_FMWK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + else () + set (${package_prefix}_INSTALL_JAR_DIR lib) + endif () + set (${package_prefix}_INSTALL_LIB_DIR lib) + endif () + if (NOT ${package_prefix}_INSTALL_INCLUDE_DIR) + set (${package_prefix}_INSTALL_INCLUDE_DIR include) + endif () + if (NOT ${package_prefix}_INSTALL_MODULE_DIR) + set (${package_prefix}_INSTALL_MODULE_DIR mod) + endif () + if (NOT ${package_prefix}_INSTALL_DATA_DIR) + if (NOT MSVC) + if (APPLE) + if (${package_prefix}_BUILD_FRAMEWORKS) + set (${package_prefix}_INSTALL_EXTRA_DIR ../SharedSupport) + else () + set (${package_prefix}_INSTALL_EXTRA_DIR share) + endif () + set (${package_prefix}_INSTALL_FWRK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + endif () + set (${package_prefix}_INSTALL_DATA_DIR share) + else () + set (${package_prefix}_INSTALL_DATA_DIR ".") + endif () + endif () + if (NOT ${package_prefix}_INSTALL_CMAKE_DIR) + set (${package_prefix}_INSTALL_CMAKE_DIR cmake) + endif () + if (NOT ${package_prefix}_INSTALL_DOC_DIR) + set (${package_prefix}_INSTALL_DOC_DIR ${${package_prefix}_INSTALL_DATA_DIR}) + endif () + message(VERBOSE "Final: ${${package_prefix}_INSTALL_DOC_DIR}") + + # Always use full RPATH, i.e. don't skip the full RPATH for the build tree + set (CMAKE_SKIP_BUILD_RPATH OFF) + # when building, don't use the install RPATH already + # (but later on when installing) + set (CMAKE_INSTALL_RPATH_USE_LINK_PATH OFF) + # add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set (CMAKE_BUILD_WITH_INSTALL_RPATH ON) + if (APPLE) + set (CMAKE_INSTALL_NAME_DIR "@rpath") + set (CMAKE_INSTALL_RPATH + "@executable_path/../${${package_prefix}_INSTALL_LIB_DIR}" + "@executable_path/" + "@loader_path/../${${package_prefix}_INSTALL_LIB_DIR}" + "@loader_path/" + ) + else () + set (CMAKE_INSTALL_RPATH "\$ORIGIN/../${${package_prefix}_INSTALL_LIB_DIR}:\$ORIGIN/") + endif () + + if (DEFINED ADDITIONAL_CMAKE_PREFIX_PATH AND EXISTS "${ADDITIONAL_CMAKE_PREFIX_PATH}") + set (CMAKE_PREFIX_PATH ${ADDITIONAL_CMAKE_PREFIX_PATH} ${CMAKE_PREFIX_PATH}) + endif () + + #set the default debug suffix for all library targets + if(NOT CMAKE_DEBUG_POSTFIX) + if (WIN32) + set (CMAKE_DEBUG_POSTFIX "_D") + else () + set (CMAKE_DEBUG_POSTFIX "_debug") + endif () + endif () + + SET_HDF_BUILD_TYPE() + + SET_HDF_OUTPUT_DIRS(${package_prefix}) + + include (FetchContent) +endmacro () + +macro (ADD_H5_FLAGS h5_flag_var infile) + file (STRINGS ${infile} TEST_FLAG_STREAM) + list (LENGTH TEST_FLAG_STREAM len_flag) + if (len_flag GREATER 0) + math (EXPR _FP_LEN "${len_flag} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + list (GET TEST_FLAG_STREAM ${line} str_flag) + string (REGEX REPLACE "^#.*" "" str_flag "${str_flag}") + if (str_flag) + list (APPEND ${h5_flag_var} "${str_flag}") + endif () + endforeach () + endif () +endmacro () diff --git a/SZ3/config/cmake/HDFPLoptions.cmake b/SZ3/config/cmake/HDFPLoptions.cmake new file mode 100755 index 00000000..3ae85937 --- /dev/null +++ b/SZ3/config/cmake/HDFPLoptions.cmake @@ -0,0 +1,21 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +##################################################################################### +#### Change default options ### +#### format: set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +##################################################################################### + +### uncomment/comment and change the following lines for other configuration options +### disable build testing +#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DH5PL_BUILD_TESTING=OFF") + +############################################################################################# diff --git a/SZ3/config/cmake/HDFPluginMacros.cmake b/SZ3/config/cmake/HDFPluginMacros.cmake new file mode 100644 index 00000000..207de025 --- /dev/null +++ b/SZ3/config/cmake/HDFPluginMacros.cmake @@ -0,0 +1,577 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +#------------------------------------------------------------------------------- +macro (BASIC_SETTINGS varname) + string (TOUPPER ${varname} PLUGIN_PACKAGE_VARNAME) + string (TOLOWER ${varname} PLUGIN_VARNAME) + set (${PLUGIN_PACKAGE_VARNAME}_PACKAGE "${PLUGIN_VARNAME}") + set (${PLUGIN_PACKAGE_VARNAME}_PACKAGE_NAME "${PLUGIN_VARNAME}") + string (TOUPPER ${${PLUGIN_PACKAGE_VARNAME}_PACKAGE_NAME} PLUGIN_PACKAGE_NAME) + string (TOLOWER ${${PLUGIN_PACKAGE_VARNAME}_PACKAGE_NAME} PLUGIN_NAME) + set (CMAKE_NO_SYSTEM_FROM_IMPORTED 1) + + #----------------------------------------------------------------------------- + # Define some CMake variables for use later in the project + #----------------------------------------------------------------------------- + set (${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR ${${PLUGIN_PACKAGE_NAME}_SOURCE_DIR}/config/cmake) + set (${PLUGIN_PACKAGE_NAME}_SRC_DIR ${${PLUGIN_PACKAGE_NAME}_SOURCE_DIR}/src) + + #----------------------------------------------------------------------------- + # Set the core names of all the libraries + #----------------------------------------------------------------------------- + set (${PLUGIN_PACKAGE_NAME}_LIB_CORENAME "${PLUGIN_NAME}") + + #----------------------------------------------------------------------------- + # Set the true names of all the libraries if customized by external project + #----------------------------------------------------------------------------- + set (${PLUGIN_PACKAGE_NAME}_LIB_NAME "${${PLUGIN_PACKAGE_NAME}_EXTERNAL_LIB_PREFIX}${${PLUGIN_PACKAGE_NAME}_LIB_CORENAME}") + + #----------------------------------------------------------------------------- + # Set the target names of all the libraries + #----------------------------------------------------------------------------- + set (${PLUGIN_PACKAGE_NAME}_LIB_TARGET ${${PLUGIN_PACKAGE_NAME}_LIB_CORENAME}) + + option (H5PL_USE_GNU_DIRS "TRUE to use GNU Coding Standard install directory variables, FALSE to use historical settings" FALSE) + if (H5PL_USE_GNU_DIRS) + include(GNUInstallDirs) + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_BIN_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR}) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_LIB_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR}/plugin) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_JAR_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_JAR_DIR ${CMAKE_INSTALL_LIBDIR}) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_INCLUDE_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_MODULE_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_MODULE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/mod) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR ${CMAKE_INSTALL_DATADIR}) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_CMAKE_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_DOC_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_DOC_DIR ${CMAKE_INSTALL_DOCDIR}) + endif () + message(STATUS "GNU: ${${PLUGIN_PACKAGE_NAME}_INSTALL_LIB_DIR}") + endif () + + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_BIN_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_BIN_DIR bin) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_LIB_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_LIB_DIR lib/plugin) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_INCLUDE_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_INCLUDE_DIR include) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR) + if (NOT MSVC) + if (APPLE) + if (${PLUGIN_PACKAGE_NAME}_BUILD_FRAMEWORKS) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_EXTRA_DIR ../SharedSupport) + else () + set (${PLUGIN_PACKAGE_NAME}_INSTALL_EXTRA_DIR share) + endif () + set (${PLUGIN_PACKAGE_NAME}_INSTALL_FWRK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + endif () + set (${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR share) + else () + set (${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR ".") + endif () + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_CMAKE_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_CMAKE_DIR cmake) + endif () + if (NOT ${PLUGIN_PACKAGE_NAME}_INSTALL_DOC_DIR) + set (${PLUGIN_PACKAGE_NAME}_INSTALL_DOC_DIR ${${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR}) + endif () + message(VERBOSE "Final: ${${PLUGIN_PACKAGE_NAME}_INSTALL_LIB_DIR}") + + #----------------------------------------------------------------------------- + # Setup output Directories + #----------------------------------------------------------------------------- + SET_HDF_OUTPUT_DIRS(${PLUGIN_PACKAGE_NAME}) + + #----------------------------------------------------------------------------- + # Targets built within this project are exported at Install time for use + # by other projects using Find${PLUGIN_PACKAGE_NAME}. + #----------------------------------------------------------------------------- + if (NOT ${PLUGIN_PACKAGE_NAME}_EXPORTED_TARGETS AND NOT "${PLUGIN_NAME}" MATCHES "h5pl") + set (${PLUGIN_PACKAGE_NAME}_EXPORTED_TARGETS "${PLUGIN_NAME}-targets") + endif () + + #----------------------------------------------------------------------------- + # To include a library in the list exported by the project AT BUILD TIME, + # add it to this variable. This is NOT used by Make Install, but for projects + # which include ${PLUGIN_NAME} as a sub-project within their build tree + #----------------------------------------------------------------------------- + set_global_variable (${PLUGIN_PACKAGE_NAME}_LIBRARIES_TO_EXPORT "") + + #----------------------------------------------------------------------------- + # Plugins are MODULEs that must be built as Shared libs + #----------------------------------------------------------------------------- + set (BUILD_SHARED_LIBS ON CACHE BOOL "Build Shared Libraries" FORCE) + set (LIB_TYPE SHARED) + add_definitions (-D${PLUGIN_PACKAGE_NAME}_BUILT_AS_DYNAMIC_LIB) + + set (CMAKE_POSITION_INDEPENDENT_CODE ON) + + if (MSVC) + set (CMAKE_MFC_FLAG 0) + endif () + + set (CMAKE_C_STANDARD 99) + set (CMAKE_C_STANDARD_REQUIRED TRUE) + + if (HDF5_BUILD_CPP_LIB) + ENABLE_LANGUAGE (CXX) + + set (CMAKE_CXX_STANDARD 98) + set (CMAKE_CXX_STANDARD_REQUIRED TRUE) + set (CMAKE_CXX_EXTENSIONS OFF) + endif () + + #----------------------------------------------------------------------------- + # Compiler specific flags : Shouldn't there be compiler tests for these + #----------------------------------------------------------------------------- + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS} -std=c99 -fomit-frame-pointer -finline-functions -fno-common") + endif () + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS} -fomit-frame-pointer -finline-functions -fno-common") + endif () + + #----------------------------------------------------------------------------- + # This is in here to help some of the GCC based IDES like Eclipse + # and code blocks parse the compiler errors and warnings better. + #----------------------------------------------------------------------------- + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmessage-length=0") + endif () + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0") + endif () + + option (DISABLE_${PLUGIN_PACKAGE_NAME}_ENCODER "build the ${PLUGIN_NAME} library without encoding" ${DISABLE_H5PL_ENCODER}) + if (DISABLE_${PLUGIN_PACKAGE_NAME}_ENCODER) + set (FILTER_DECODE_ONLY 1) + endif () + + #----------------------------------------------------------------------------- + # Set includes needed for build + #----------------------------------------------------------------------------- + set (${PLUGIN_PACKAGE_NAME}_INCLUDES_BUILD_TIME + ${${PLUGIN_PACKAGE_NAME}_SRC_DIR} ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR} + ) +endmacro () + +macro (HDF5_SUPPORT link_hdf) + set (CMAKE_MODULE_PATH ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR} ${CMAKE_MODULE_PATH}) + set (CMAKE_POSITION_INDEPENDENT_CODE ON) + # plugins must use SHARED libraries + option (USE_SHARED_LIBS "Use Shared Libraries" ON) + + if (NOT H5PL_HDF5_HEADER) + if (USE_SHARED_LIBS) + set (FIND_HDF_COMPONENTS C shared) + else () + set (FIND_HDF_COMPONENTS C static) + endif () + message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") + set (SEARCH_PACKAGE_NAME ${HDF5_PACKAGE_NAME}) + + find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) + message (STATUS "HDF5 C libs:${HDF5_FOUND} static:${HDF5_static_C_FOUND} and shared:${HDF5_shared_C_FOUND}") + if (HDF5_FOUND) + if (NOT HDF5_static_C_FOUND AND NOT HDF5_shared_C_FOUND) + #find library from non-dual-binary package + set (FIND_HDF_COMPONENTS C) + message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") + + find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) + message (STATUS "HDF5 libs:${HDF5_FOUND} C:${HDF5_C_FOUND}") + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_LIBRARIES}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_LIBRARIES}) + if (HDF5_BUILD_SHARED_LIBS) + add_definitions (-DH5_BUILT_AS_DYNAMIC_LIB) + else () + add_definitions (-DH5_BUILT_AS_STATIC_LIB) + endif () + endif () + if (USE_SHARED_LIBS AND WIN32) + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dumpdll") + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repackdll") + else () + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump") + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack") + endif () + set (H5PL_HDF5_DUMP_EXECUTABLE $) + set (H5PL_HDF5_REPACK_EXECUTABLE $) + else () + get_filename_component (_LIBRARY_PATH ${HDF5_INCLUDE_DIR} DIRECTORY) + set (HDF5_LIBRARY_PATH "${_LIBRARY_PATH}/lib") + if (USE_SHARED_LIBS AND HDF5_shared_C_FOUND) + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_C_SHARED_LIBRARY}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_C_SHARED_LIBRARY}) + endif () + else () + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_C_STATIC_LIBRARY}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_C_STATIC_LIBRARY}) + endif () + endif () + if (HDF5_VERSION VERSION_LESS "1.14.4" AND NOT HDF5_shared_C_FOUND) + if (NOT TARGET ${HDF5_NAMESPACE}h5dump-shared) + add_executable (${HDF5_NAMESPACE}h5dump-shared IMPORTED) + endif () + if (NOT TARGET ${HDF5_NAMESPACE}h5repack-shared) + add_executable (${HDF5_NAMESPACE}h5repack-shared IMPORTED) + endif () + set (H5PL_HDF5_DUMP_EXECUTABLE $) + set (H5PL_HDF5_REPACK_EXECUTABLE $) + else () + if (NOT TARGET ${HDF5_NAMESPACE}h5dump) + add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) + endif() + if (NOT TARGET ${HDF5_NAMESPACE}h5repack) + add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) + endif() + set (H5PL_HDF5_DUMP_EXECUTABLE $) + set (H5PL_HDF5_REPACK_EXECUTABLE $) + endif() + endif () + # Determine if a threading package is available on this system + if (HDF5_ENABLE_THREADS) + find_package (Threads) + if (NOT Threads_FOUND) + message (FATAL_ERROR " **** thread support requires C11 threads, Win32 threads or Pthreads **** ") + endif () + endif () + else () + find_package (HDF5) # Legacy find + #Legacy find_package does not set HDF5_TOOLS_DIR, so we set it here + set (HDF5_TOOLS_DIR ${HDF5_LIBRARY_DIRS}/../bin) + #Legacy find_package does not set HDF5_BUILD_SHARED_LIBS, so we set it here + if (USE_SHARED_LIBS AND EXISTS "${HDF5_LIBRARY_DIRS}/libhdf5.so") + set (HDF5_BUILD_SHARED_LIBS 1) + else () + set (HDF5_BUILD_SHARED_LIBS 0) + endif () + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_LIBRARIES}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_LIBRARIES}) + endif () + add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump") + set (H5PL_HDF5_DUMP_EXECUTABLE $) + add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack") + set (H5PL_HDF5_REPACK_EXECUTABLE $) + endif () + + set (HDF5_PACKAGE_NAME ${SEARCH_PACKAGE_NAME}) + + if (HDF5_FOUND) + set (H5PL_HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIR}) + set (H5PL_HDF5_HAVE_H5PUBCONF_H 1) + set (H5PL_HDF5_HAVE_HDF5 1) + set (H5PL_HDF5_HEADER "h5pubconf.h") + message (STATUS "HDF5-${HDF5_VERSION_STRING} found: INC=${HDF5_INCLUDE_DIR} TOOLS=${HDF5_TOOLS_DIR}") + else () + message (FATAL_ERROR " HDF5 is Required for plugin library") + endif () + else () + # This project is being called from within another and HDF5 is already configured + set (H5PL_HDF5_HAVE_H5PUBCONF_H 1) + set (H5PL_HDF5_HAVE_HDF5 1) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${H5PL_HDF5_LINK_LIBS}) + endif () + set (USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE) + endif () + message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}") +endmacro () + +macro (INSTALL_SUPPORT varname) + string(TOUPPER ${varname} PLUGIN_PACKAGE_NAME) + string(TOLOWER ${varname} PLUGIN_NAME) + + if (${PLUGIN_PACKAGE_NAME}_CPACK_ENABLE) + #----------------------------------------------------------------------------- + # Set variables needed for installation + #----------------------------------------------------------------------------- + set (${PLUGIN_PACKAGE_NAME}_VERSION_STRING ${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION}) + set (${PLUGIN_PACKAGE_NAME}_VERSION_MAJOR ${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION_MAJOR}) + set (${PLUGIN_PACKAGE_NAME}_VERSION_MINOR ${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION_MINOR}) + + #----------------------------------------------------------------------------- + # Configure the H5PL_Examples.cmake file and the examples + #----------------------------------------------------------------------------- + if (${PLUGIN_PACKAGE_NAME}_BUILD_TESTING) + configure_file ( + ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/H5PL_Examples.cmake.in + ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/H5PL_Examples.cmake @ONLY + ) + install ( + FILES + ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/H5PL_Examples.cmake + ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/CTestScript.cmake + ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/HDFPLoptions.cmake + DESTINATION ${${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR} + COMPONENT hdfdocuments + ) + execute_process( + COMMAND ${CMAKE_COMMAND} -E copy_directory ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/binex ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/HDFPLExamples + ) + install ( + DIRECTORY ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/HDFPLExamples + DESTINATION ${${PLUGIN_PACKAGE_NAME}_INSTALL_DATA_DIR} + USE_SOURCE_PERMISSIONS + COMPONENT hdfdocuments + ) + endif () + + #----------------------------------------------------------------------------- + # Configure the README.txt file for the binary package + #----------------------------------------------------------------------------- + PLUGIN_README_PROPERTIES (${PLUGIN_PACKAGE_NAME}) + + #----------------------------------------------------------------------------- + # Configure the COPYING.txt file for the windows binary package + #----------------------------------------------------------------------------- + if (WIN32) + configure_file (${CMAKE_SOURCE_DIR}/COPYING ${CMAKE_BINARY_DIR}/COPYING.txt @ONLY) + endif () + + #----------------------------------------------------------------------------- + # Add Document File(s) to CMake Install + #----------------------------------------------------------------------------- + install ( + FILES + ${${PLUGIN_PACKAGE_NAME}_SOURCE_DIR}/docs/PluginLibraries.txt + DESTINATION ${${PLUGIN_PACKAGE_NAME}_INSTALL_DOC_DIR} + COMPONENT hdfdocuments + ) + + #----------------------------------------------------------------------------- + # Check for Installation Utilities + #----------------------------------------------------------------------------- + if (WIN32) + set (PF_ENV_EXT "(x86)") + find_program (NSIS_EXECUTABLE NSIS.exe PATHS "$ENV{ProgramFiles}\\NSIS" "$ENV{ProgramFiles${PF_ENV_EXT}}\\NSIS") + if(NOT CPACK_WIX_ROOT) + file(TO_CMAKE_PATH "$ENV{WIX}" CPACK_WIX_ROOT) + endif() + find_program (WIX_EXECUTABLE candle PATHS "${CPACK_WIX_ROOT}/bin") + endif () + + if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + if (CMAKE_HOST_UNIX) + set (CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}/HDF_Group/${${PLUGIN_PACKAGE_NAME}_PACKAGE_NAME}/${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION}" + CACHE PATH "Install path prefix, prepended onto install directories." FORCE + ) + else () + GetDefaultWindowsPrefixBase(CMAKE_GENERIC_PROGRAM_FILES) + set (CMAKE_INSTALL_PREFIX + "${CMAKE_GENERIC_PROGRAM_FILES}/HDF_Group/${${PLUGIN_PACKAGE_NAME}_PACKAGE_NAME}/${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION}" + CACHE PATH "Install path prefix, prepended onto install directories." FORCE + ) + set (CMAKE_GENERIC_PROGRAM_FILES) + endif () + endif () + + #----------------------------------------------------------------------------- + # Set the cpack variables + #----------------------------------------------------------------------------- + if (NOT ${PLUGIN_PACKAGE_NAME}_EXTERNALLY_CONFIGURED) + set (CPACK_PACKAGE_VENDOR "HDF_Group") + set (CPACK_PACKAGE_NAME "${${PLUGIN_PACKAGE_NAME}_PACKAGE_NAME}") + if (NOT WIN32 OR ${PLUGIN_PACKAGE_NAME}_VERS_SUBRELEASE MATCHES "^[0-9]+$") + set (CPACK_PACKAGE_VERSION "${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION_STRING}") + else () + set (CPACK_PACKAGE_VERSION "${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION}") + endif () + set (CPACK_PACKAGE_VERSION_MAJOR "${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION_MAJOR}") + set (CPACK_PACKAGE_VERSION_MINOR "${${PLUGIN_PACKAGE_NAME}_PACKAGE_VERSION_MINOR}") + set (CPACK_PACKAGE_VERSION_PATCH "") + set (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/COPYING") + set (CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_BINARY_DIR}/README.txt") + set (CPACK_RESOURCE_FILE_README "${CMAKE_BINARY_DIR}/README.txt") + set (CPACK_PACKAGE_RELOCATABLE TRUE) + set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PLUGIN_NAME} Installation") + if (H5PL_OVERRIDE_VERSION) + string(TOUPPER ${HDF5_PACKAGE_NAME} PLUGIN_HDF5_PACKAGE_NAME) + set (CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_VENDOR}/${PLUGIN_HDF5_PACKAGE_NAME}/${H5PL_OVERRIDE_VERSION}") + else () + set (CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_VENDOR}/${CPACK_PACKAGE_NAME}/${CPACK_PACKAGE_VERSION}") + endif () + + set (CPACK_ORIG_SOURCE_DIR ${CMAKE_SOURCE_DIR}) + if ("$ENV{BINSIGN}" STREQUAL "exists") + set (CPACK_PRE_BUILD_SCRIPTS ${CMAKE_SOURCE_DIR}/config/cmake/SignPackageFiles.cmake) + endif () + + set (CPACK_GENERATOR "TGZ") + if (WIN32) + set (CPACK_GENERATOR "ZIP") + + if (NSIS_EXECUTABLE) + list (APPEND CPACK_GENERATOR "NSIS") + endif () + set (CPACK_NSIS_PACKAGE_NAME "${${PLUGIN_PACKAGE_NAME}_PACKAGE_STRING}") + if (CMAKE_CL_64) + set (CPACK_NSIS_INSTALL_ROOT "$PROGRAMFILES64") + set (CPACK_PACKAGE_INSTALL_REGISTRY_KEY "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION} (Win64)") + else () + set (CPACK_NSIS_INSTALL_ROOT "$PROGRAMFILES") + set (CPACK_PACKAGE_INSTALL_REGISTRY_KEY "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}") + endif () + # set the install/unistall icon used for the installer itself + # There is a bug in NSI that does not handle full unix paths properly. + #set (CPACK_NSIS_MUI_ICON "${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}\\\\${PLUGIN_PACKAGE_NAME}.ico") + #set (CPACK_NSIS_MUI_UNIICON "${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}\\\\${PLUGIN_PACKAGE_NAME}.ico") + # set the package header icon for MUI + #set (CPACK_PACKAGE_ICON "${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}\\\\${PLUGIN_PACKAGE_NAME}.bmp") + set (CPACK_NSIS_DISPLAY_NAME "${CPACK_NSIS_PACKAGE_NAME}, is a dynamic filter for HDF5") + #set (CPACK_PACKAGE_INSTALL_DIRECTORY "plugins") + set (CPACK_NSIS_CONTACT "${${PLUGIN_PACKAGE_NAME}_PACKAGE_BUGREPORT}") + set (CPACK_NSIS_MODIFY_PATH ON) + + if (WIX_EXECUTABLE) + list (APPEND CPACK_GENERATOR "WIX") + endif () + # WiX variables + set (CPACK_WIX_UNINSTALL "1") + set (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_BINARY_DIR}/COPYING.txt") +# set (CPACK_WIX_PRODUCT_ICON "${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}\\\\${PLUGIN_PACKAGE_NAME}.ico") + elseif (APPLE) + list (APPEND CPACK_GENERATOR "STGZ") + list (APPEND CPACK_GENERATOR "DragNDrop") + set (CPACK_COMPONENTS_ALL_IN_ONE_PACKAGE ON) + set (CPACK_PACKAGING_INSTALL_PREFIX "/${CPACK_PACKAGE_INSTALL_DIRECTORY}") +# set (CPACK_PACKAGE_ICON "${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/h5pl.gif") + + option (${PLUGIN_PACKAGE_NAME}_PACK_MACOSX_BUNDLE "Package the ${PLUGIN_PACKAGE_NAME} Library in a Bundle" OFF) + if (${PLUGIN_PACKAGE_NAME}_PACK_MACOSX_BUNDLE) + list (APPEND CPACK_GENERATOR "Bundle") + set (CPACK_BUNDLE_NAME "${${PLUGIN_PACKAGE_NAME}_PACKAGE_STRING}") + set (CPACK_BUNDLE_LOCATION "/") # make sure CMAKE_INSTALL_PREFIX ends in / + set (CMAKE_INSTALL_PREFIX "/${CPACK_BUNDLE_NAME}.framework/Versions/${CPACK_PACKAGE_VERSION}/${CPACK_PACKAGE_NAME}/") +# set (CPACK_BUNDLE_ICON "${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/H5PL.icns") +# set (CPACK_BUNDLE_PLIST "${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/CMakeFiles/Info.plist") + set (CPACK_APPLE_GUI_INFO_STRING "${PLUGIN_PACKAGE_NAME} Plugin Library") + set (CPACK_APPLE_GUI_COPYRIGHT "Copyright © ???. All rights reserved.") + set (CPACK_SHORT_VERSION_STRING "${CPACK_PACKAGE_VERSION}") + set (CPACK_APPLE_GUI_BUNDLE_NAME "${${PLUGIN_PACKAGE_NAME}_PACKAGE_STRING}") + set (CPACK_APPLE_GUI_VERSION_STRING "${CPACK_PACKAGE_VERSION_STRING}") + set (CPACK_APPLE_GUI_SHORT_VERSION_STRING "${CPACK_PACKAGE_VERSION}") + #----------------------------------------------------------------------------- + # Configure the Info.plist file for the install bundle + #----------------------------------------------------------------------------- +# configure_file ( +# ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/CPack.Info.plist.in +# ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/CMakeFiles/Info.plist @ONLY +# ) +# configure_file ( +# ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/PkgInfo.in +# ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/CMakeFiles/PkgInfo @ONLY +# ) +# configure_file ( +# ${${PLUGIN_PACKAGE_NAME}_RESOURCES_DIR}/version.plist.in +# ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/CMakeFiles/version.plist @ONLY +# ) +# install ( +# FILES ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/CMakeFiles/PkgInfo +# ${${PLUGIN_PACKAGE_NAME}_BINARY_DIR}/CMakeFiles/version.plist +# DESTINATION .. +# ) + endif () + else () + list (APPEND CPACK_GENERATOR "STGZ") + set (CPACK_PACKAGING_INSTALL_PREFIX "/${CPACK_PACKAGE_INSTALL_DIRECTORY}") + set (CPACK_COMPONENTS_ALL_IN_ONE_PACKAGE ON) + + set (CPACK_DEBIAN_PACKAGE_SECTION "Libraries") + set (CPACK_DEBIAN_PACKAGE_MAINTAINER "${${PLUGIN_PACKAGE_NAME}_PACKAGE_BUGREPORT}") + + find_program (DPKGSHLIB_EXE dpkg-shlibdeps) + if (DPKGSHLIB_EXE) + list (APPEND CPACK_GENERATOR "DEB") + set (CPACK_DEBIAN_PACKAGE_SECTION "Libraries") + set (CPACK_DEBIAN_PACKAGE_MAINTAINER "${H5PL_PACKAGE_BUGREPORT}") + endif () + + find_program (RPMBUILD_EXE rpmbuild) + if (RPMBUILD_EXE) + list (APPEND CPACK_GENERATOR "RPM") + set (CPACK_RPM_PACKAGE_RELEASE "1") + set (CPACK_RPM_PACKAGE_RELEASE_DIST ON) + set (CPACK_RPM_COMPONENT_INSTALL ON) + set (CPACK_RPM_PACKAGE_RELOCATABLE ON) + set (CPACK_RPM_FILE_NAME "RPM-DEFAULT") + set (CPACK_RPM_PACKAGE_NAME "${CPACK_PACKAGE_NAME}") + set (CPACK_RPM_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION}") + set (CPACK_RPM_PACKAGE_VENDOR "${CPACK_PACKAGE_VENDOR}") + set (CPACK_RPM_PACKAGE_LICENSE "BSD-style") + set (CPACK_RPM_PACKAGE_GROUP "Development/Libraries") + set (CPACK_RPM_PACKAGE_URL "${H5PL_PACKAGE_URL}") + set (CPACK_RPM_PACKAGE_SUMMARY "HDF5 Plugins are a suite of filters supporting registered compression libraries.") + set (CPACK_RPM_PACKAGE_DESCRIPTION + "The HDF5 technology suite includes: + + * A versatile data model that can represent very complex data objects and a wide variety of metadata. + + * A completely portable file format with no limit on the number or size of data objects in the collection. + + * A software library that runs on a range of computational platforms, from laptops to massively parallel systems, and implements a high-level API with C, C++, Fortran 90, and Java interfaces. + + * A rich set of integrated performance features that allow for access time and storage space optimizations. + +The HDF5 data model, file format, API, library, and tools are open and distributed without charge. +" + ) + endif () + endif () + set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${${PLUGIN_PACKAGE_NAME}_BINARY_DIR};${PLUGIN_NAME};ALL;/") + set (CPACK_ALL_INSTALL_TYPES Full User) + set (CPACK_INSTALL_TYPE_FULL_DISPLAY_NAME "Everything") + endif () + + include (CPack) + + cpack_add_component_group(Runtime) + + cpack_add_component (libraries + DISPLAY_NAME "${PLUGIN_NAME} Libraries" + GROUP Runtime + ) + cpack_add_component_group(hdfdocuments + DISPLAY_NAME "${PLUGIN_NAME} Documents" + GROUP Documents + INSTALL_TYPES Full User + ) + endif () +endmacro () diff --git a/SZ3/config/cmake/NSIS.InstallOptions.ini.in b/SZ3/config/cmake/NSIS.InstallOptions.ini.in new file mode 100644 index 00000000..ede3cc99 --- /dev/null +++ b/SZ3/config/cmake/NSIS.InstallOptions.ini.in @@ -0,0 +1,37 @@ +[Settings] +NumFields=4 + +[Field 1] +Type=label +Text=By default @CPACK_PACKAGE_NAME@ does NOT add its directory to the system PATH. +Left=0 +Right=-1 +Top=0 +Bottom=20 + +[Field 2] +Type=radiobutton +Text=Do not add @CPACK_PACKAGE_NAME@ to the system PATH +Left=0 +Right=-1 +Top=30 +Bottom=40 +State=1 + +[Field 3] +Type=radiobutton +Text=Add @CPACK_PACKAGE_NAME@ to the system PATH for all users +Left=0 +Right=-1 +Top=40 +Bottom=50 +State=0 + +[Field 4] +Type=radiobutton +Text=Add @CPACK_PACKAGE_NAME@ to the system PATH for current user +Left=0 +Right=-1 +Top=50 +Bottom=60 +State=0 diff --git a/SZ3/config/cmake/NSIS.template.in b/SZ3/config/cmake/NSIS.template.in new file mode 100644 index 00000000..db794664 --- /dev/null +++ b/SZ3/config/cmake/NSIS.template.in @@ -0,0 +1,974 @@ +; CPack install script designed for a nmake build + +;-------------------------------- +; You must define these values + + !define VERSION "@CPACK_PACKAGE_VERSION@" + !define PATCH "@CPACK_PACKAGE_VERSION_PATCH@" + !define INST_DIR "@CPACK_TEMPORARY_DIRECTORY@" + +;-------------------------------- +;Variables + + Var MUI_TEMP + Var STARTMENU_FOLDER + Var SV_ALLUSERS + Var START_MENU + Var DO_NOT_ADD_TO_PATH + Var ADD_TO_PATH_ALL_USERS + Var ADD_TO_PATH_CURRENT_USER + Var INSTALL_DESKTOP + Var IS_DEFAULT_INSTALLDIR +;-------------------------------- +;Include Modern UI + + !include "MUI.nsh" + + ;Default installation folder + InstallDir "@CPACK_NSIS_INSTALL_ROOT@\@CPACK_PACKAGE_INSTALL_DIRECTORY@" + +;-------------------------------- +;General + + ;Name and file + Name "@CPACK_NSIS_PACKAGE_NAME@" + OutFile "@CPACK_TOPLEVEL_DIRECTORY@/@CPACK_OUTPUT_FILE_NAME@" + + ;Set compression + SetCompressor @CPACK_NSIS_COMPRESSOR@ + +@CPACK_NSIS_DEFINES@ + + !include Sections.nsh + +;--- Component support macros: --- +; The code for the add/remove functionality is from: +; http://nsis.sourceforge.net/Add/Remove_Functionality +; It has been modified slightly and extended to provide +; inter-component dependencies. +Var AR_SecFlags +Var AR_RegFlags +@CPACK_NSIS_SECTION_SELECTED_VARS@ + +; Loads the "selected" flag for the section named SecName into the +; variable VarName. +!macro LoadSectionSelectedIntoVar SecName VarName + SectionGetFlags ${${SecName}} $${VarName} + IntOp $${VarName} $${VarName} & ${SF_SELECTED} ;Turn off all other bits +!macroend + +; Loads the value of a variable... can we get around this? +!macro LoadVar VarName + IntOp $R0 0 + $${VarName} +!macroend + +; Sets the value of a variable +!macro StoreVar VarName IntValue + IntOp $${VarName} 0 + ${IntValue} +!macroend + +!macro InitSection SecName + ; This macro reads component installed flag from the registry and + ;changes checked state of the section on the components page. + ;Input: section index constant name specified in Section command. + + ClearErrors + ;Reading component status from registry + ReadRegDWORD $AR_RegFlags HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@\Components\${SecName}" "Installed" + IfErrors "default_${SecName}" + ;Status will stay default if registry value not found + ;(component was never installed) + IntOp $AR_RegFlags $AR_RegFlags & ${SF_SELECTED} ;Turn off all other bits + SectionGetFlags ${${SecName}} $AR_SecFlags ;Reading default section flags + IntOp $AR_SecFlags $AR_SecFlags & 0xFFFE ;Turn lowest (enabled) bit off + IntOp $AR_SecFlags $AR_RegFlags | $AR_SecFlags ;Change lowest bit + + ; Note whether this component was installed before + !insertmacro StoreVar ${SecName}_was_installed $AR_RegFlags + IntOp $R0 $AR_RegFlags & $AR_RegFlags + + ;Writing modified flags + SectionSetFlags ${${SecName}} $AR_SecFlags + + "default_${SecName}:" + !insertmacro LoadSectionSelectedIntoVar ${SecName} ${SecName}_selected +!macroend + +!macro FinishSection SecName + ; This macro reads section flag set by user and removes the section + ;if it is not selected. + ;Then it writes component installed flag to registry + ;Input: section index constant name specified in Section command. + + SectionGetFlags ${${SecName}} $AR_SecFlags ;Reading section flags + ;Checking lowest bit: + IntOp $AR_SecFlags $AR_SecFlags & ${SF_SELECTED} + IntCmp $AR_SecFlags 1 "leave_${SecName}" + ;Section is not selected: + ;Calling Section uninstall macro and writing zero installed flag + !insertmacro "Remove_${${SecName}}" + WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@\Components\${SecName}" \ + "Installed" 0 + Goto "exit_${SecName}" + + "leave_${SecName}:" + ;Section is selected: + WriteRegDWORD HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@\Components\${SecName}" \ + "Installed" 1 + + "exit_${SecName}:" +!macroend + +!macro RemoveSection_CPack SecName + ; This macro is used to call section's Remove_... macro + ;from the uninstaller. + ;Input: section index constant name specified in Section command. + + !insertmacro "Remove_${${SecName}}" +!macroend + +; Determine whether the selection of SecName changed +!macro MaybeSelectionChanged SecName + !insertmacro LoadVar ${SecName}_selected + SectionGetFlags ${${SecName}} $R1 + IntOp $R1 $R1 & ${SF_SELECTED} ;Turn off all other bits + + ; See if the status has changed: + IntCmp $R0 $R1 "${SecName}_unchanged" + !insertmacro LoadSectionSelectedIntoVar ${SecName} ${SecName}_selected + + IntCmp $R1 ${SF_SELECTED} "${SecName}_was_selected" + !insertmacro "Deselect_required_by_${SecName}" + goto "${SecName}_unchanged" + + "${SecName}_was_selected:" + !insertmacro "Select_${SecName}_depends" + + "${SecName}_unchanged:" +!macroend +;--- End of Add/Remove macros --- + +;-------------------------------- +;Interface Settings + + !define MUI_HEADERIMAGE + !define MUI_ABORTWARNING + +;-------------------------------- +; path functions + +!verbose 3 +!include "WinMessages.NSH" +!verbose 4 + +;---------------------------------------- +; based upon a script of "Written by KiCHiK 2003-01-18 05:57:02" +;---------------------------------------- +!verbose 3 +!include "WinMessages.NSH" +!verbose 4 +;==================================================== +; get_NT_environment +; Returns: the selected environment +; Output : head of the stack +;==================================================== +!macro select_NT_profile UN +Function ${UN}select_NT_profile + StrCmp $ADD_TO_PATH_ALL_USERS "1" 0 environment_single + DetailPrint "Selected environment for all users" + Push "all" + Return + environment_single: + DetailPrint "Selected environment for current user only." + Push "current" + Return +FunctionEnd +!macroend +!insertmacro select_NT_profile "" +!insertmacro select_NT_profile "un." +;---------------------------------------------------- +!define NT_current_env 'HKCU "Environment"' +!define NT_all_env 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"' + +!ifndef WriteEnvStr_RegKey + !ifdef ALL_USERS + !define WriteEnvStr_RegKey \ + 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"' + !else + !define WriteEnvStr_RegKey 'HKCU "Environment"' + !endif +!endif + +; AddToPath - Adds the given dir to the search path. +; Input - head of the stack +; Note - Win9x systems requires reboot + +Function AddToPath + Exch $0 + Push $1 + Push $2 + Push $3 + + # don't add if the path doesn't exist + IfFileExists "$0\*.*" "" AddToPath_done + + ReadEnvStr $1 PATH + ; if the path is too long for a NSIS variable NSIS will return a 0 + ; length string. If we find that, then warn and skip any path + ; modification as it will trash the existing path. + StrLen $2 $1 + IntCmp $2 0 CheckPathLength_ShowPathWarning CheckPathLength_Done CheckPathLength_Done + CheckPathLength_ShowPathWarning: + Messagebox MB_OK|MB_ICONEXCLAMATION "Warning! PATH too long installer unable to modify PATH!" + Goto AddToPath_done + CheckPathLength_Done: + Push "$1;" + Push "$0;" + Call StrStr + Pop $2 + StrCmp $2 "" "" AddToPath_done + Push "$1;" + Push "$0\;" + Call StrStr + Pop $2 + StrCmp $2 "" "" AddToPath_done + GetFullPathName /SHORT $3 $0 + Push "$1;" + Push "$3;" + Call StrStr + Pop $2 + StrCmp $2 "" "" AddToPath_done + Push "$1;" + Push "$3\;" + Call StrStr + Pop $2 + StrCmp $2 "" "" AddToPath_done + + Call IsNT + Pop $1 + StrCmp $1 1 AddToPath_NT + ; Not on NT + StrCpy $1 $WINDIR 2 + FileOpen $1 "$1\autoexec.bat" a + FileSeek $1 -1 END + FileReadByte $1 $2 + IntCmp $2 26 0 +2 +2 # DOS EOF + FileSeek $1 -1 END # write over EOF + FileWrite $1 "$\r$\nSET PATH=%PATH%;$3$\r$\n" + FileClose $1 + SetRebootFlag true + Goto AddToPath_done + + AddToPath_NT: + StrCmp $ADD_TO_PATH_ALL_USERS "1" ReadAllKey + ReadRegStr $1 ${NT_current_env} "PATH" + Goto DoTrim + ReadAllKey: + ReadRegStr $1 ${NT_all_env} "PATH" + DoTrim: + StrCmp $1 "" AddToPath_NTdoIt + Push $1 + Call Trim + Pop $1 + StrCpy $0 "$1;$0" + AddToPath_NTdoIt: + StrCmp $ADD_TO_PATH_ALL_USERS "1" WriteAllKey + WriteRegExpandStr ${NT_current_env} "PATH" $0 + Goto DoSend + WriteAllKey: + WriteRegExpandStr ${NT_all_env} "PATH" $0 + DoSend: + SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 + + AddToPath_done: + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + + +; RemoveFromPath - Remove a given dir from the path +; Input: head of the stack + +Function un.RemoveFromPath + Exch $0 + Push $1 + Push $2 + Push $3 + Push $4 + Push $5 + Push $6 + + IntFmt $6 "%c" 26 # DOS EOF + + Call un.IsNT + Pop $1 + StrCmp $1 1 unRemoveFromPath_NT + ; Not on NT + StrCpy $1 $WINDIR 2 + FileOpen $1 "$1\autoexec.bat" r + GetTempFileName $4 + FileOpen $2 $4 w + GetFullPathName /SHORT $0 $0 + StrCpy $0 "SET PATH=%PATH%;$0" + Goto unRemoveFromPath_dosLoop + + unRemoveFromPath_dosLoop: + FileRead $1 $3 + StrCpy $5 $3 1 -1 # read last char + StrCmp $5 $6 0 +2 # if DOS EOF + StrCpy $3 $3 -1 # remove DOS EOF so we can compare + StrCmp $3 "$0$\r$\n" unRemoveFromPath_dosLoopRemoveLine + StrCmp $3 "$0$\n" unRemoveFromPath_dosLoopRemoveLine + StrCmp $3 "$0" unRemoveFromPath_dosLoopRemoveLine + StrCmp $3 "" unRemoveFromPath_dosLoopEnd + FileWrite $2 $3 + Goto unRemoveFromPath_dosLoop + unRemoveFromPath_dosLoopRemoveLine: + SetRebootFlag true + Goto unRemoveFromPath_dosLoop + + unRemoveFromPath_dosLoopEnd: + FileClose $2 + FileClose $1 + StrCpy $1 $WINDIR 2 + Delete "$1\autoexec.bat" + CopyFiles /SILENT $4 "$1\autoexec.bat" + Delete $4 + Goto unRemoveFromPath_done + + unRemoveFromPath_NT: + StrCmp $ADD_TO_PATH_ALL_USERS "1" unReadAllKey + ReadRegStr $1 ${NT_current_env} "PATH" + Goto unDoTrim + unReadAllKey: + ReadRegStr $1 ${NT_all_env} "PATH" + unDoTrim: + StrCpy $5 $1 1 -1 # copy last char + StrCmp $5 ";" +2 # if last char != ; + StrCpy $1 "$1;" # append ; + Push $1 + Push "$0;" + Call un.StrStr ; Find `$0;` in $1 + Pop $2 ; pos of our dir + StrCmp $2 "" unRemoveFromPath_done + ; else, it is in path + # $0 - path to add + # $1 - path var + StrLen $3 "$0;" + StrLen $4 $2 + StrCpy $5 $1 -$4 # $5 is now the part before the path to remove + StrCpy $6 $2 "" $3 # $6 is now the part after the path to remove + StrCpy $3 $5$6 + + StrCpy $5 $3 1 -1 # copy last char + StrCmp $5 ";" 0 +2 # if last char == ; + StrCpy $3 $3 -1 # remove last char + + StrCmp $ADD_TO_PATH_ALL_USERS "1" unWriteAllKey + WriteRegExpandStr ${NT_current_env} "PATH" $3 + Goto unDoSend + unWriteAllKey: + WriteRegExpandStr ${NT_all_env} "PATH" $3 + unDoSend: + SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 + + unRemoveFromPath_done: + Pop $6 + Pop $5 + Pop $4 + Pop $3 + Pop $2 + Pop $1 + Pop $0 +FunctionEnd + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Uninstall sutff +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +########################################### +# Utility Functions # +########################################### + +;==================================================== +; IsNT - Returns 1 if the current system is NT, 0 +; otherwise. +; Output: head of the stack +;==================================================== +; IsNT +; no input +; output, top of the stack = 1 if NT or 0 if not +; +; Usage: +; Call IsNT +; Pop $R0 +; ($R0 at this point is 1 or 0) + +!macro IsNT un +Function ${un}IsNT + Push $0 + ReadRegStr $0 HKLM "SOFTWARE\Microsoft\Windows NT\CurrentVersion" CurrentVersion + StrCmp $0 "" 0 IsNT_yes + ; we are not NT. + Pop $0 + Push 0 + Return + + IsNT_yes: + ; NT!!! + Pop $0 + Push 1 +FunctionEnd +!macroend +!insertmacro IsNT "" +!insertmacro IsNT "un." + +; StrStr +; input, top of stack = string to search for +; top of stack-1 = string to search in +; output, top of stack (replaces with the portion of the string remaining) +; modifies no other variables. +; +; Usage: +; Push "this is a long ass string" +; Push "ass" +; Call StrStr +; Pop $R0 +; ($R0 at this point is "ass string") + +!macro StrStr un +Function ${un}StrStr +Exch $R1 ; st=haystack,old$R1, $R1=needle + Exch ; st=old$R1,haystack + Exch $R2 ; st=old$R1,old$R2, $R2=haystack + Push $R3 + Push $R4 + Push $R5 + StrLen $R3 $R1 + StrCpy $R4 0 + ; $R1=needle + ; $R2=haystack + ; $R3=len(needle) + ; $R4=cnt + ; $R5=tmp + loop: + StrCpy $R5 $R2 $R3 $R4 + StrCmp $R5 $R1 done + StrCmp $R5 "" done + IntOp $R4 $R4 + 1 + Goto loop +done: + StrCpy $R1 $R2 "" $R4 + Pop $R5 + Pop $R4 + Pop $R3 + Pop $R2 + Exch $R1 +FunctionEnd +!macroend +!insertmacro StrStr "" +!insertmacro StrStr "un." + +Function Trim ; Added by Pelaca + Exch $R1 + Push $R2 +Loop: + StrCpy $R2 "$R1" 1 -1 + StrCmp "$R2" " " RTrim + StrCmp "$R2" "$\n" RTrim + StrCmp "$R2" "$\r" RTrim + StrCmp "$R2" ";" RTrim + GoTo Done +RTrim: + StrCpy $R1 "$R1" -1 + Goto Loop +Done: + Pop $R2 + Exch $R1 +FunctionEnd + +Function ConditionalAddToRegisty + Pop $0 + Pop $1 + StrCmp "$0" "" ConditionalAddToRegisty_EmptyString + WriteRegStr SHCTX "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" \ + "$1" "$0" + ;MessageBox MB_OK "Set Registry: '$1' to '$0'" + DetailPrint "Set install registry entry: '$1' to '$0'" + ConditionalAddToRegisty_EmptyString: +FunctionEnd + +;-------------------------------- + +!ifdef CPACK_USES_DOWNLOAD +Function DownloadFile + IfFileExists $INSTDIR\* +2 + CreateDirectory $INSTDIR + Pop $0 + + ; Skip if already downloaded + IfFileExists $INSTDIR\$0 0 +2 + Return + + StrCpy $1 "@CPACK_DOWNLOAD_SITE@" + + try_again: + NSISdl::download "$1/$0" "$INSTDIR\$0" + + Pop $1 + StrCmp $1 "success" success + StrCmp $1 "Cancelled" cancel + MessageBox MB_OK "Download failed: $1" + cancel: + Return + success: +FunctionEnd +!endif + +;-------------------------------- +; Installation types +@CPACK_NSIS_INSTALLATION_TYPES@ + +;-------------------------------- +; Component sections +@CPACK_NSIS_COMPONENT_SECTIONS@ + +;-------------------------------- +; Define some macro setting for the gui +@CPACK_NSIS_INSTALLER_MUI_ICON_CODE@ +@CPACK_NSIS_INSTALLER_ICON_CODE@ +@CPACK_NSIS_INSTALLER_MUI_COMPONENTS_DESC@ +@CPACK_NSIS_INSTALLER_MUI_FINISHPAGE_RUN_CODE@ + +;-------------------------------- +;Pages + !insertmacro MUI_PAGE_WELCOME + + !insertmacro MUI_PAGE_LICENSE "@CPACK_RESOURCE_FILE_LICENSE@" + Page custom InstallOptionsPage + !insertmacro MUI_PAGE_DIRECTORY + + ;Start Menu Folder Page Configuration + !define MUI_STARTMENUPAGE_REGISTRY_ROOT "SHCTX" + !define MUI_STARTMENUPAGE_REGISTRY_KEY "Software\@CPACK_PACKAGE_VENDOR@\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" + !define MUI_STARTMENUPAGE_REGISTRY_VALUENAME "Start Menu Folder" + !insertmacro MUI_PAGE_STARTMENU Application $STARTMENU_FOLDER + + @CPACK_NSIS_PAGE_COMPONENTS@ + + !insertmacro MUI_PAGE_INSTFILES + !insertmacro MUI_PAGE_FINISH + + !insertmacro MUI_UNPAGE_CONFIRM + !insertmacro MUI_UNPAGE_INSTFILES + +;-------------------------------- +;Languages + + !insertmacro MUI_LANGUAGE "English" ;first language is the default language + !insertmacro MUI_LANGUAGE "Albanian" + !insertmacro MUI_LANGUAGE "Arabic" + !insertmacro MUI_LANGUAGE "Basque" + !insertmacro MUI_LANGUAGE "Belarusian" + !insertmacro MUI_LANGUAGE "Bosnian" + !insertmacro MUI_LANGUAGE "Breton" + !insertmacro MUI_LANGUAGE "Bulgarian" + !insertmacro MUI_LANGUAGE "Croatian" + !insertmacro MUI_LANGUAGE "Czech" + !insertmacro MUI_LANGUAGE "Danish" + !insertmacro MUI_LANGUAGE "Dutch" + !insertmacro MUI_LANGUAGE "Estonian" + !insertmacro MUI_LANGUAGE "Farsi" + !insertmacro MUI_LANGUAGE "Finnish" + !insertmacro MUI_LANGUAGE "French" + !insertmacro MUI_LANGUAGE "German" + !insertmacro MUI_LANGUAGE "Greek" + !insertmacro MUI_LANGUAGE "Hebrew" + !insertmacro MUI_LANGUAGE "Hungarian" + !insertmacro MUI_LANGUAGE "Icelandic" + !insertmacro MUI_LANGUAGE "Indonesian" + !insertmacro MUI_LANGUAGE "Irish" + !insertmacro MUI_LANGUAGE "Italian" + !insertmacro MUI_LANGUAGE "Japanese" + !insertmacro MUI_LANGUAGE "Korean" + !insertmacro MUI_LANGUAGE "Kurdish" + !insertmacro MUI_LANGUAGE "Latvian" + !insertmacro MUI_LANGUAGE "Lithuanian" + !insertmacro MUI_LANGUAGE "Luxembourgish" + !insertmacro MUI_LANGUAGE "Macedonian" + !insertmacro MUI_LANGUAGE "Malay" + !insertmacro MUI_LANGUAGE "Mongolian" + !insertmacro MUI_LANGUAGE "Norwegian" + !insertmacro MUI_LANGUAGE "Polish" + !insertmacro MUI_LANGUAGE "Portuguese" + !insertmacro MUI_LANGUAGE "PortugueseBR" + !insertmacro MUI_LANGUAGE "Romanian" + !insertmacro MUI_LANGUAGE "Russian" + !insertmacro MUI_LANGUAGE "Serbian" + !insertmacro MUI_LANGUAGE "SerbianLatin" + !insertmacro MUI_LANGUAGE "SimpChinese" + !insertmacro MUI_LANGUAGE "Slovak" + !insertmacro MUI_LANGUAGE "Slovenian" + !insertmacro MUI_LANGUAGE "Spanish" + !insertmacro MUI_LANGUAGE "Swedish" + !insertmacro MUI_LANGUAGE "Thai" + !insertmacro MUI_LANGUAGE "TradChinese" + !insertmacro MUI_LANGUAGE "Turkish" + !insertmacro MUI_LANGUAGE "Ukrainian" + !insertmacro MUI_LANGUAGE "Welsh" + + +;-------------------------------- +;Reserve Files + + ;These files should be inserted before other files in the data block + ;Keep these lines before any File command + ;Only for solid compression (by default, solid compression is enabled for BZIP2 and LZMA) + + ReserveFile "NSIS.InstallOptions.ini" + !insertmacro MUI_RESERVEFILE_INSTALLOPTIONS + +;-------------------------------- +;Installer Sections + +Section "-Core installation" + ;Use the entire tree produced by the INSTALL target. Keep the + ;list of directories here in sync with the RMDir commands below. + SetOutPath "$INSTDIR" + @CPACK_NSIS_EXTRA_PREINSTALL_COMMANDS@ + @CPACK_NSIS_FULL_INSTALL@ + + ;Store installation folder + WriteRegStr SHCTX "Software\@CPACK_PACKAGE_VENDOR@\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "" $INSTDIR + + ;Create uninstaller + WriteUninstaller "$INSTDIR\Uninstall.exe" + Push "DisplayName" + Push "@CPACK_NSIS_DISPLAY_NAME@" + Call ConditionalAddToRegisty + Push "DisplayVersion" + Push "@CPACK_PACKAGE_VERSION@" + Call ConditionalAddToRegisty + Push "Publisher" + Push "@CPACK_PACKAGE_VENDOR@" + Call ConditionalAddToRegisty + Push "UninstallString" + Push "$INSTDIR\Uninstall.exe" + Call ConditionalAddToRegisty + Push "NoRepair" + Push "1" + Call ConditionalAddToRegisty + + !ifdef CPACK_NSIS_ADD_REMOVE + ;Create add/remove functionality + Push "ModifyPath" + Push "$INSTDIR\AddRemove.exe" + Call ConditionalAddToRegisty + !else + Push "NoModify" + Push "1" + Call ConditionalAddToRegisty + !endif + + ; Optional registration + Push "DisplayIcon" + Push "$INSTDIR\@CPACK_NSIS_INSTALLED_ICON_NAME@" + Call ConditionalAddToRegisty + Push "HelpLink" + Push "@CPACK_NSIS_HELP_LINK@" + Call ConditionalAddToRegisty + Push "URLInfoAbout" + Push "@CPACK_NSIS_URL_INFO_ABOUT@" + Call ConditionalAddToRegisty + Push "Contact" + Push "@CPACK_NSIS_CONTACT@" + Call ConditionalAddToRegisty + !insertmacro MUI_INSTALLOPTIONS_READ $INSTALL_DESKTOP "NSIS.InstallOptions.ini" "Field 5" "State" + !insertmacro MUI_STARTMENU_WRITE_BEGIN Application + + ;Create shortcuts + CreateDirectory "$SMPROGRAMS\$STARTMENU_FOLDER" +@CPACK_NSIS_CREATE_ICONS@ +@CPACK_NSIS_CREATE_ICONS_EXTRA@ + CreateShortCut "$SMPROGRAMS\$STARTMENU_FOLDER\Uninstall.lnk" "$INSTDIR\Uninstall.exe" + + ;Read a value from an InstallOptions INI file + !insertmacro MUI_INSTALLOPTIONS_READ $DO_NOT_ADD_TO_PATH "NSIS.InstallOptions.ini" "Field 2" "State" + !insertmacro MUI_INSTALLOPTIONS_READ $ADD_TO_PATH_ALL_USERS "NSIS.InstallOptions.ini" "Field 3" "State" + !insertmacro MUI_INSTALLOPTIONS_READ $ADD_TO_PATH_CURRENT_USER "NSIS.InstallOptions.ini" "Field 4" "State" + + ; Write special uninstall registry entries + Push "StartMenu" + Push "$STARTMENU_FOLDER" + Call ConditionalAddToRegisty + Push "DoNotAddToPath" + Push "$DO_NOT_ADD_TO_PATH" + Call ConditionalAddToRegisty + Push "AddToPathAllUsers" + Push "$ADD_TO_PATH_ALL_USERS" + Call ConditionalAddToRegisty + Push "AddToPathCurrentUser" + Push "$ADD_TO_PATH_CURRENT_USER" + Call ConditionalAddToRegisty + Push "InstallToDesktop" + Push "$INSTALL_DESKTOP" + Call ConditionalAddToRegisty + + !insertmacro MUI_STARTMENU_WRITE_END + +@CPACK_NSIS_EXTRA_INSTALL_COMMANDS@ + +SectionEnd + +Section "-Add to path" + Push $INSTDIR\bin + StrCmp "@CPACK_NSIS_MODIFY_PATH@" "ON" 0 doNotAddToPath + StrCmp $DO_NOT_ADD_TO_PATH "1" doNotAddToPath 0 + Call AddToPath + doNotAddToPath: +SectionEnd + +;-------------------------------- +; Create custom pages +Function InstallOptionsPage + !insertmacro MUI_HEADER_TEXT "Install Options" "Choose options for installing @CPACK_NSIS_PACKAGE_NAME@" + !insertmacro MUI_INSTALLOPTIONS_DISPLAY "NSIS.InstallOptions.ini" + +FunctionEnd + +;-------------------------------- +; determine admin versus local install +Function un.onInit + + ClearErrors + UserInfo::GetName + IfErrors noLM + Pop $0 + UserInfo::GetAccountType + Pop $1 + StrCmp $1 "Admin" 0 +3 + SetShellVarContext all + ;MessageBox MB_OK 'User "$0" is in the Admin group' + Goto done + StrCmp $1 "Power" 0 +3 + SetShellVarContext all + ;MessageBox MB_OK 'User "$0" is in the Power Users group' + Goto done + + noLM: + ;Get installation folder from registry if available + + done: + +FunctionEnd + +;--- Add/Remove callback functions: --- +!macro SectionList MacroName + ;This macro used to perform operation on multiple sections. + ;List all of your components in following manner here. +@CPACK_NSIS_COMPONENT_SECTION_LIST@ +!macroend + +Section -FinishComponents + ;Removes unselected components and writes component status to registry + !insertmacro SectionList "FinishSection" + +!ifdef CPACK_NSIS_ADD_REMOVE + ; Get the name of the installer executable + System::Call 'kernel32::GetModuleFileNameA(i 0, t .R0, i 1024) i r1' + StrCpy $R3 $R0 + + ; Strip off the last 13 characters, to see if we have AddRemove.exe + StrLen $R1 $R0 + IntOp $R1 $R0 - 13 + StrCpy $R2 $R0 13 $R1 + StrCmp $R2 "AddRemove.exe" addremove_installed + + ; We're not running AddRemove.exe, so install it + CopyFiles $R3 $INSTDIR\AddRemove.exe + + addremove_installed: +!endif +SectionEnd +;--- End of Add/Remove callback functions --- + +;-------------------------------- +; Component dependencies +Function .onSelChange + !insertmacro SectionList MaybeSelectionChanged +FunctionEnd + +;-------------------------------- +;Uninstaller Section + +Section "Uninstall" + ReadRegStr $START_MENU SHCTX \ + "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "StartMenu" + ;MessageBox MB_OK "Start menu is in: $START_MENU" + ReadRegStr $DO_NOT_ADD_TO_PATH SHCTX \ + "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "DoNotAddToPath" + ReadRegStr $ADD_TO_PATH_ALL_USERS SHCTX \ + "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "AddToPathAllUsers" + ReadRegStr $ADD_TO_PATH_CURRENT_USER SHCTX \ + "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "AddToPathCurrentUser" + ;MessageBox MB_OK "Add to path: $DO_NOT_ADD_TO_PATH all users: $ADD_TO_PATH_ALL_USERS" + ReadRegStr $INSTALL_DESKTOP SHCTX \ + "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "InstallToDesktop" + ;MessageBox MB_OK "Install to desktop: $INSTALL_DESKTOP " + +@CPACK_NSIS_EXTRA_UNINSTALL_COMMANDS@ + + ;Remove files we installed. + ;Keep the list of directories here in sync with the File commands above. +@CPACK_NSIS_DELETE_FILES@ +@CPACK_NSIS_DELETE_DIRECTORIES@ + +!ifdef CPACK_NSIS_ADD_REMOVE + ;Remove the add/remove program + Delete "$INSTDIR\AddRemove.exe" +!endif + + ;Remove the uninstaller itself. + Delete "$INSTDIR\Uninstall.exe" + DeleteRegKey SHCTX "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" + + ;Remove the installation directory if it is empty. + RMDir "$INSTDIR" + + ; Remove the registry entries. + DeleteRegKey SHCTX "Software\@CPACK_PACKAGE_VENDOR@\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" + + ; Removes all optional components + !insertmacro SectionList "RemoveSection_CPack" + + !insertmacro MUI_STARTMENU_GETFOLDER Application $MUI_TEMP + + Delete "$SMPROGRAMS\$MUI_TEMP\Uninstall.lnk" +@CPACK_NSIS_DELETE_ICONS@ +@CPACK_NSIS_DELETE_ICONS_EXTRA@ + + ;Delete empty start menu parent directories + StrCpy $MUI_TEMP "$SMPROGRAMS\$MUI_TEMP" + + startMenuDeleteLoop: + ClearErrors + RMDir $MUI_TEMP + GetFullPathName $MUI_TEMP "$MUI_TEMP\.." + + IfErrors startMenuDeleteLoopDone + + StrCmp "$MUI_TEMP" "$SMPROGRAMS" startMenuDeleteLoopDone startMenuDeleteLoop + startMenuDeleteLoopDone: + + ; If the user changed the shortcut, then untinstall may not work. This should + ; try to fix it. + StrCpy $MUI_TEMP "$START_MENU" + Delete "$SMPROGRAMS\$MUI_TEMP\Uninstall.lnk" +@CPACK_NSIS_DELETE_ICONS_EXTRA@ + + ;Delete empty start menu parent directories + StrCpy $MUI_TEMP "$SMPROGRAMS\$MUI_TEMP" + + secondStartMenuDeleteLoop: + ClearErrors + RMDir $MUI_TEMP + GetFullPathName $MUI_TEMP "$MUI_TEMP\.." + + IfErrors secondStartMenuDeleteLoopDone + + StrCmp "$MUI_TEMP" "$SMPROGRAMS" secondStartMenuDeleteLoopDone secondStartMenuDeleteLoop + secondStartMenuDeleteLoopDone: + + DeleteRegKey /ifempty SHCTX "Software\@CPACK_PACKAGE_VENDOR@\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" + + Push $INSTDIR\bin + StrCmp $DO_NOT_ADD_TO_PATH_ "1" doNotRemoveFromPath 0 + Call un.RemoveFromPath + doNotRemoveFromPath: +SectionEnd + +;-------------------------------- +; determine admin versus local install +; Is install for "AllUsers" or "JustMe"? +; Default to "JustMe" - set to "AllUsers" if admin or on Win9x +; This function is used for the very first "custom page" of the installer. +; This custom page does not show up visibly, but it executes prior to the +; first visible page and sets up $INSTDIR properly... +; Choose different default installation folder based on SV_ALLUSERS... +; "Program Files" for AllUsers, "My Documents" for JustMe... + +Function .onInit + StrCmp "@CPACK_NSIS_ENABLE_UNINSTALL_BEFORE_INSTALL@" "ON" 0 inst + + ReadRegStr $0 HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\@CPACK_PACKAGE_INSTALL_REGISTRY_KEY@" "UninstallString" + StrCmp $0 "" inst + + MessageBox MB_YESNOCANCEL|MB_ICONEXCLAMATION \ + "@CPACK_NSIS_PACKAGE_NAME@ is already installed. $\n$\nDo you want to uninstall the old version before installing the new one?" \ + IDYES uninst IDNO inst + Abort + +;Run the uninstaller +uninst: + ClearErrors + StrLen $2 "\Uninstall.exe" + StrCpy $3 $0 -$2 # remove "\Uninstall.exe" from UninstallString to get path + ExecWait '$0 _?=$3' ;Do not copy the uninstaller to a temp file + + IfErrors uninst_failed inst +uninst_failed: + MessageBox MB_OK|MB_ICONSTOP "Uninstall failed." + Abort + + +inst: + ; Reads components status for registry + !insertmacro SectionList "InitSection" + + ; check to see if /D has been used to change + ; the install directory by comparing it to the + ; install directory that is expected to be the + ; default + StrCpy $IS_DEFAULT_INSTALLDIR 0 + StrCmp "$INSTDIR" "@CPACK_NSIS_INSTALL_ROOT@\@CPACK_PACKAGE_INSTALL_DIRECTORY@" 0 +2 + StrCpy $IS_DEFAULT_INSTALLDIR 1 + + StrCpy $SV_ALLUSERS "JustMe" + ; if default install dir then change the default + ; if it is installed for JustMe + StrCmp "$IS_DEFAULT_INSTALLDIR" "1" 0 +2 + StrCpy $INSTDIR "$DOCUMENTS\@CPACK_PACKAGE_INSTALL_DIRECTORY@" + + ClearErrors + UserInfo::GetName + IfErrors noLM + Pop $0 + UserInfo::GetAccountType + Pop $1 + StrCmp $1 "Admin" 0 +4 + SetShellVarContext all + ;MessageBox MB_OK 'User "$0" is in the Admin group' + StrCpy $SV_ALLUSERS "AllUsers" + Goto done + StrCmp $1 "Power" 0 +4 + SetShellVarContext all + ;MessageBox MB_OK 'User "$0" is in the Power Users group' + StrCpy $SV_ALLUSERS "AllUsers" + Goto done + + noLM: + StrCpy $SV_ALLUSERS "AllUsers" + ;Get installation folder from registry if available + + done: + StrCmp $SV_ALLUSERS "AllUsers" 0 +3 + StrCmp "$IS_DEFAULT_INSTALLDIR" "1" 0 +2 + StrCpy $INSTDIR "@CPACK_NSIS_INSTALL_ROOT@\@CPACK_PACKAGE_INSTALL_DIRECTORY@" + + StrCmp "@CPACK_NSIS_MODIFY_PATH@" "ON" 0 noOptionsPage + !insertmacro MUI_INSTALLOPTIONS_EXTRACT "NSIS.InstallOptions.ini" + + noOptionsPage: +FunctionEnd diff --git a/SZ3/config/cmake/README.txt.cmake.in b/SZ3/config/cmake/README.txt.cmake.in new file mode 100644 index 00000000..2faf049f --- /dev/null +++ b/SZ3/config/cmake/README.txt.cmake.in @@ -0,0 +1,105 @@ +@H5SZ3_PACKAGE_NAME@ version @H5SZ3_PACKAGE_VERSION_STRING@ +------------------------------------------------------------------------------ + +This directory contains the binary (release) distribution of +@H5SZ3_PACKAGE_NAME@-@H5SZ3_PACKAGE_VERSION_MAJOR@.@H5SZ3_PACKAGE_VERSION_MINOR@ that was compiled on; + @BINARY_PLATFORM@. + +It was built with the following options: + -- @LIB_TYPE@ C libraries + -- @LIB_TYPE@ H5SZ3 tools + -- @H5_LIB_TYPE@ SZ3 @SZ3_VERSION_STRING@ + -- HDF5 @HDF5_VERSION_STRING@ + +The contents of this directory are: + + COPYING - Copyright notice + README.txt - This file + @H5SZ3_PACKAGE_NAME@-@H5SZ3_PACKAGE_VERSION@-@BINARY_SYSTEM_NAME@.@BINARY_INSTALL_ENDING@ - H5SZ3 Install Utility + +Installation +=========================================================================== +The customary location for the plugins is under the hdf5 libraries location. + The plugins depend on the environment variable, HDF5_PLUGIN_PATH, and can + be installed into any location. +On linux, the installer should be executed in the location to install the + plugin. The plugins will install into the location: + /HDF_Group/HDF5/1.@H5SZ3_PACKAGE_VERSION_MINOR@/lib/plugin +On Windows, the installer will default to the same location as the hdf5 installer: + C:/Program Files/HDF_Group/HDF5/1.@H5SZ3_PACKAGE_VERSION_MINOR@ and into the /lib/plugin dir. +1. Execute @H5SZ3_PACKAGE_NAME@-@H5SZ3_PACKAGE_VERSION@-@BINARY_SYSTEM_NAME@.@BINARY_INSTALL_ENDING@ +2. Follow prompts +=========================================================================== + +After Installation, Test using CMake +=========================================================================== +The HDF5 Plugins examples expect a HDF5 library built with CMake, but can be + used with an autotools built binary. Normally the plugins are installed + with the HDF5 library. + +Copy the following dir and file from the share dir in the H5PL INSTALL location + to your working dir: + HDFPLExamples + H5PL_Examples.cmake + +Execute the ctest script: + ctest -S H5PL_Examples.cmake,HDF5INSTALLDIR= -C Release -VV -O test.log + where has the following dirs; + on windows: + ------bin + ------include + ------lib + ------cmake + on linux: + ------bin + ------include + ------lib + ------share +------------------------------------------------------------------------------ + +If you do not use the ctest script, you must supply the following environment + variables: + HDF5_ROOT + HDF5_PLUGIN_PATH + LD_LIBRARY_PATH + and make and cd into a build folder: + and execute: + cmake -C ../HDFPLExamples/config/cmake/cacheinit.cmake \ + -DCMAKE_BUILD_TYPE:STRING=Release -G "Unix Makefiles" \ + ../HDFPLExamples + and execute: + cmake --build . --config Release + and execute: + ctest . -C Release + +=========================================================================== + +After Installation, Test using h5cc on linux +=========================================================================== +The compression library assumes that HDF5 is installed to the directory +pointed to by the environment variable HDF5_HOME, e.g. HDF5_HOME=/usr/local/hdf5. + +It can be installed in user space or admin space, as long as this variable is +defined properly. + +The installed version of HDF5 must have been built with ‐‐enable‐static=no in +order to force the hdf5 tools to use dynamic linking, which is required by the plugin. + +By default, HDF5 looks for plugins to be installed in $HDF5_HOME/lib/plugin. +Be sure to copy the shared plugins libraries (*.so files) from this install +to the $HDF5_HOME/lib/plugin folder. + +Copy the following dir from the share dir in the H5PL INSTALL location + to your working dir: + HDFPLExamples + +Change your working dir to the HDFPLExamples/example/test folder. + +Execute the test.sh script. + +=========================================================================== + +Documentation for this release can be found at the following URL: + https://support.hdfgroup.org/documentation/index.html + +Bugs should be reported to help@hdfgroup.org. diff --git a/SZ3/config/cmake/SignPackageFiles.cmake b/SZ3/config/cmake/SignPackageFiles.cmake new file mode 100644 index 00000000..0e18c7a2 --- /dev/null +++ b/SZ3/config/cmake/SignPackageFiles.cmake @@ -0,0 +1,43 @@ +# This script signs the targets for the package +message(STATUS "Signing script in ${CPACK_TEMPORARY_INSTALL_DIRECTORY} and ${CPACK_PACKAGE_INSTALL_DIRECTORY}") + +# RPM needs ALL_COMPONENTS_IN_ONE added to path between ${CPACK_TEMPORARY_INSTALL_DIRECTORY} and ${CPACK_PACKAGE_INSTALL_DIRECTORY} +if (CPACK_GENERATOR MATCHES "RPM") + set (CPACK_TARGET_FILE_DIRECTORY "${CPACK_TEMPORARY_INSTALL_DIRECTORY}/ALL_COMPONENTS_IN_ONE/${CPACK_PACKAGE_INSTALL_DIRECTORY}") +elseif (CPACK_GENERATOR MATCHES "WIX" OR CPACK_GENERATOR MATCHES "NSIS") + set (CPACK_TARGET_FILE_DIRECTORY "${CPACK_TEMPORARY_INSTALL_DIRECTORY}/libraries") +elseif (CPACK_GENERATOR MATCHES "ZIP") + set (CPACK_TARGET_FILE_DIRECTORY "${CPACK_TEMPORARY_INSTALL_DIRECTORY}") +elseif (CPACK_GENERATOR MATCHES "DragNDrop") + set (CPACK_TARGET_FILE_DIRECTORY "${CPACK_TEMPORARY_INSTALL_DIRECTORY}/ALL_IN_ONE/${CPACK_PACKAGE_INSTALL_DIRECTORY}") +else () + set (CPACK_TARGET_FILE_DIRECTORY "${CPACK_TEMPORARY_INSTALL_DIRECTORY}/${CPACK_PACKAGE_INSTALL_DIRECTORY}") +endif () +file (GLOB target_list LIST_DIRECTORIES false "${CPACK_TARGET_FILE_DIRECTORY}/lib/plugin/*.*") +foreach (targetfile IN LISTS target_list) + if (WIN32) + # Sign the targets + execute_process (COMMAND $ENV{SIGNTOOLDIR}/signtool + sign /v /debug /fd SHA256 /tr http://timestamp.acs.microsoft.com /td SHA256 + /dlib "Microsoft.Trusted.Signing.Client/bin/x64/Azure.CodeSigning.Dlib.dll" /dmdf ${CPACK_ORIG_SOURCE_DIR}/credentials.json + ${targetfile} + ) + execute_process ( + COMMAND ${CMAKE_COMMAND} -E echo "Signing the target ${targetfile}" + ) + elseif (APPLE) + # Sign the targets + execute_process (COMMAND codesign + --force --timestamp --options runtime --entitlements ${CPACK_ORIG_SOURCE_DIR}/config/cmake/distribution.entitlements + --verbose=4 --strict --sign "$ENV{SIGNER}" + ${targetfile} + ) + execute_process ( + COMMAND ${CMAKE_COMMAND} -E echo "Signing the target ${targetfile}" + ) + else () + execute_process ( + COMMAND ${CMAKE_COMMAND} -E echo "Signing the target ${targetfile}" + ) + endif () +endforeach () diff --git a/SZ3/config/cmake/binex/CMakeLists.txt b/SZ3/config/cmake/binex/CMakeLists.txt new file mode 100644 index 00000000..0065ab72 --- /dev/null +++ b/SZ3/config/cmake/binex/CMakeLists.txt @@ -0,0 +1,47 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +cmake_minimum_required (VERSION 3.18) +project (HDFPLExamples C) + +#----------------------------------------------------------------------------- +# For standard build of HDF5 plugin examples. +#----------------------------------------------------------------------------- +include (${PROJECT_SOURCE_DIR}/config/cmake/HDFPluginMacros.cmake) +BASIC_SETTINGS (HDFPLExamples) + +#----------------------------------------------------------------------------- +# HDF5 support +#----------------------------------------------------------------------------- +HDF5_SUPPORT (TRUE) +message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS}") + +#----------------------------------------------------------------------------- +# Dashboard and Testing Settings +#----------------------------------------------------------------------------- +option (H5PL_BUILD_TESTING "Build HDFPL Example Testing" OFF) +if (H5PL_BUILD_TESTING) + set (DART_TESTING_TIMEOUT 1200 + CACHE STRING + "Timeout in seconds for each test (default 1200=20minutes)" + ) + enable_testing () + include (CTest) + include (${PROJECT_SOURCE_DIR}/CTestConfig.cmake) + configure_file (${H5PL_RESOURCES_DIR}/CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.ctest @ONLY) +endif () + +#----------------------------------------------------------------------------- +# Build examples +#----------------------------------------------------------------------------- +if (H5PL_BUILD_EXAMPLES) + add_subdirectory (example) +endif () diff --git a/SZ3/config/cmake/binex/CMakePresets.json b/SZ3/config/cmake/binex/CMakePresets.json new file mode 100644 index 00000000..255c2c86 --- /dev/null +++ b/SZ3/config/cmake/binex/CMakePresets.json @@ -0,0 +1,226 @@ +{ + "version": 6, + "include": [ + "config/cmake-presets/hidden-presets.json" + ], + "configurePresets": [ + { + "name": "ci-base-examples", + "hidden": true, + "cacheVariables": { + "HDF5_NAMESPACE": {"type": "STRING", "value": "hdf5::"}, + "HDF5_PACKAGE_NAME": {"type": "STRING", "value": "hdf5"}, + "H5PL_BUILD_TESTING": "ON" + } + }, + { + "name": "ci-StdShar", + "hidden": true, + "inherits": ["ci-base", "ci-base-examples"], + "cacheVariables": { + "BUILD_SHARED_LIBS": "ON", + "USE_SHARED_LIBS": "ON" + } + }, + { + "name": "ci-StdShar-MSVC", + "description": "MSVC Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-MSVC", + "ci-StdShar" + ] + }, + { + "name": "ci-StdShar-Clang", + "description": "Clang Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-Clang", + "ci-StdShar" + ] + }, + { + "name": "ci-StdShar-GNUC", + "description": "GNUC Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-GNUC", + "ci-StdShar" + ] + }, + { + "name": "ci-StdShar-macos-Clang", + "description": "Clang Standard Config for macos (Release)", + "inherits": [ + "ci-macos-arm64-Release-Clang", + "ci-StdShar" + ] + }, + { + "name": "ci-StdShar-macos-GNUC", + "description": "GNUC Standard Config for macos (Release)", + "inherits": [ + "ci-macos-arm64-Release-GNUC", + "ci-StdShar" + ] + }, + { + "name": "ci-StdShar-Intel", + "description": "Intel Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-Intel", + "ci-StdShar" + ] + } + ], + "buildPresets": [ + { + "name": "ci-StdShar-MSVC", + "description": "MSVC Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-MSVC", + "inherits": [ + "ci-x64-Release-MSVC" + ] + }, + { + "name": "ci-StdShar-Clang", + "description": "Clang Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-Clang", + "inherits": [ + "ci-x64-Release-Clang" + ] + }, + { + "name": "ci-StdShar-GNUC", + "description": "GNUC Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-GNUC", + "verbose": true, + "inherits": [ + "ci-x64-Release-GNUC" + ] + }, + { + "name": "ci-StdShar-macos-Clang", + "description": "Clang Standard Build for macos-arm64 (Release)", + "configurePreset": "ci-StdShar-macos-Clang", + "inherits": [ + "ci-macos-arm64-Release-Clang" + ] + }, + { + "name": "ci-StdShar-macos-GNUC", + "description": "GNUC Standard Build for macos-arm64 (Release)", + "configurePreset": "ci-StdShar-macos-GNUC", + "verbose": true, + "inherits": [ + "ci-macos-arm64-Release-GNUC" + ] + }, + { + "name": "ci-StdShar-Intel", + "description": "Intel Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-Intel", + "verbose": true, + "inherits": [ + "ci-x64-Release-Intel" + ] + } + ], + "testPresets": [ + { + "name": "ci-StdShar-MSVC", + "configurePreset": "ci-StdShar-MSVC", + "inherits": [ + "ci-x64-Release-MSVC" + ] + }, + { + "name": "ci-StdShar-Clang", + "configurePreset": "ci-StdShar-Clang", + "inherits": [ + "ci-x64-Release-Clang" + ] + }, + { + "name": "ci-StdShar-macos-Clang", + "configurePreset": "ci-StdShar-macos-Clang", + "inherits": [ + "ci-macos-arm64-Release-Clang" + ], + "execution": { + "noTestsAction": "error", + "timeout": 180, + "jobs": 2 + } + }, + { + "name": "ci-StdShar-macos-GNUC", + "configurePreset": "ci-StdShar-macos-GNUC", + "inherits": [ + "ci-macos-arm64-Release-GNUC" + ] + }, + { + "name": "ci-StdShar-GNUC", + "configurePreset": "ci-StdShar-GNUC", + "inherits": [ + "ci-x64-Release-GNUC" + ] + }, + { + "name": "ci-StdShar-Intel", + "configurePreset": "ci-StdShar-Intel", + "inherits": [ + "ci-x64-Release-Intel" + ] + } + ], + "workflowPresets": [ + { + "name": "ci-StdShar-MSVC", + "steps": [ + {"type": "configure", "name": "ci-StdShar-MSVC"}, + {"type": "build", "name": "ci-StdShar-MSVC"}, + {"type": "test", "name": "ci-StdShar-MSVC"} + ] + }, + { + "name": "ci-StdShar-Clang", + "steps": [ + {"type": "configure", "name": "ci-StdShar-Clang"}, + {"type": "build", "name": "ci-StdShar-Clang"}, + {"type": "test", "name": "ci-StdShar-Clang"} + ] + }, + { + "name": "ci-StdShar-macos-Clang", + "steps": [ + {"type": "configure", "name": "ci-StdShar-macos-Clang"}, + {"type": "build", "name": "ci-StdShar-macos-Clang"}, + {"type": "test", "name": "ci-StdShar-macos-Clang"} + ] + }, + { + "name": "ci-StdShar-GNUC", + "steps": [ + {"type": "configure", "name": "ci-StdShar-GNUC"}, + {"type": "build", "name": "ci-StdShar-GNUC"}, + {"type": "test", "name": "ci-StdShar-GNUC"} + ] + }, + { + "name": "ci-StdShar-macos-GNUC", + "steps": [ + {"type": "configure", "name": "ci-StdShar-macos-GNUC"}, + {"type": "build", "name": "ci-StdShar-macos-GNUC"}, + {"type": "test", "name": "ci-StdShar-macos-GNUC"} + ] + }, + { + "name": "ci-StdShar-Intel", + "steps": [ + {"type": "configure", "name": "ci-StdShar-Intel"}, + {"type": "build", "name": "ci-StdShar-Intel"}, + {"type": "test", "name": "ci-StdShar-Intel"} + ] + } + ] +} diff --git a/SZ3/config/cmake/binex/CTestConfig.cmake b/SZ3/config/cmake/binex/CTestConfig.cmake new file mode 100644 index 00000000..c12a08a6 --- /dev/null +++ b/SZ3/config/cmake/binex/CTestConfig.cmake @@ -0,0 +1,29 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +## This file should be placed in the root directory of your project. +## Then modify the CMakeLists.txt file in the root directory of your +## project to incorporate the testing dashboard. +## # The following are required to uses Dart and the Cdash dashboard +## ENABLE_TESTING() +## INCLUDE(CTest) +set (CTEST_PROJECT_NAME "HDF5PLUGINS") +set (CTEST_NIGHTLY_START_TIME "18:00:00 CST") + +set (CTEST_DROP_METHOD "https") +set (CTEST_DROP_SITE "cdash.hdfgroup.org") +set (CTEST_DROP_LOCATION "/submit.php?project=HDF5PLUGINS") +set (CTEST_DROP_SITE_CDASH TRUE) + +set (UPDATE_TYPE git) + +set (CTEST_TESTING_TIMEOUT 1200) +set (DART_TESTING_TIMEOUT 1200) diff --git a/SZ3/config/cmake/binex/Using_CMake.txt b/SZ3/config/cmake/binex/Using_CMake.txt new file mode 100644 index 00000000..12db8667 --- /dev/null +++ b/SZ3/config/cmake/binex/Using_CMake.txt @@ -0,0 +1,259 @@ +************************************************************************ +* Build and Test HDF5PL Examples with CMake * +************************************************************************ + +Notes: This short instruction is written for users who want to quickly build + HDFPL Examples using the HDFPL binary package using the CMake tools. + The following instructions will show the default usage and then present + common changes for non-default installations. + + More information about using CMake can be found at the Kitware + site, www.cmake.org. + + Table of Contents + +Section I: Preconditions +Section II: Building HDF5 Examples with CMake + 1. Using presets + 2. Using CTestScript.cmake + 3. Using Command Line CMake +Section III: Defaults in the CMakePresets.json file +Section IV: Defaults in the HDF5PL_Examples_options.cmake file + + +======================================================================== +I. Preconditions +======================================================================== + + 1. We suggest you obtain the latest CMake from the Kitware + web site. The HDF5PL product requires a minimum CMake version + of 3.18. If you are using VS2022, the minimum CMake version is 3.21. + CMakePresets.json, requires CMake 3.25 or higher. + + 2. You have installed the HDF5 library built with CMake, by executing + the HDF Install Utility (the *.msi file in the binary package for + Windows or the *.sh on Linux). You can obtain pre-built binaries + from The HDF Group's website at www.hdfgroup.org. + + 3. Set the HDF5_ROOT CMake variable, -DHDF5_ROOT= + or environment variable, set(ENV{HDF5_ROOT} "") + to the installed location of HDF5. + On Windows: + HDF5_ROOT=C:/Program Files/HDF_Group/HDF5/z.y.x/ + On unix: + HDF5_ROOT=/HDF_Group/HDF5/z.y.x/ + + If you are using shared libraries, you may need to add to the path + environment variable. Set the path environment variable to the + installed location of the library files for HDF5. + On Windows (*.dll): + PATH=%PATH%;C:/Program Files/HDF_Group/HDF5/z.y.x/bin + On unix (*.so): + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/HDF_Group/HDF5/z.y.x/lib + + (Note there are no quote characters used on Windows and all platforms + use forward slashes) + + 4. Set the environment variable HDF5_PLUGIN_PATH to the installed location of + the library files for HDFPL. + On Windows: + HDF5_PLUGIN_PATH=C:/Program Files/HDF_Group/HDF5/z.y.x/lib/plugin + On unix: + HDF5_PLUGIN_PATH=/HDF_Group/HDF5/z.y.x/lib/plugin + + 5. Use separate source and build directories. + (CMake commands are executed in the build directory) + +======================================================================== +II. Building HDF5 Plugin Examples with CMake +======================================================================== + + 1. Using presets + Files in the HDF5 Plugins install/HDF5PLExamples directory: + CMakePresets.json + + Default build process: + Create a directory to run the examples, i.e. \test_hdf5. + Copy the HDF5PLExamples folder to this directory. + Change into the HDF5PLExamples directory where CMakePresets.json exists. + + Other options can be changed by creating a + CMakeUsersPresets.json file. (Advanced usage) + + Available configurations presets can be displayed by executing: + cmake -S --list-presets + + Using individual command presets (where is GNUC or MSVC or Clang): + cmake --preset ci-StdShar- + cmake --build --preset ci-StdShar- + ctest --preset ci-StdShar- + cpack --preset ci-StdShar- + + Using the workflow preset to configure, build, and testthe standard configuration: + execute "cmake --workflow --preset ci-StdShar- --fresh" + where is GNUC or MSVC or Clang + + 2. Using CTestScript.cmake + Files in the HDF5 install directory: + HDF5PLExamples folder + CTestScript.cmake + HDF5PL_Examples.cmake + HDF5PLoptions.cmake + + Default build process: + Create a directory to run the examples, i.e. \test_hdf5. + Copy HDF5PLExamples folder to this directory. + Copy CTestScript.cmake to this directory. + Copy HDF5PL_Examples.cmake to this directory. + Copy HDF5PLoptions.cmake to this directory. + The default source folder is defined as "HDF5PLExamples". It can be changed + with the CTEST_SOURCE_NAME script option. + The default installation folder is defined for the platform. + It can be changed with the INSTALLDIR script option. + (Note: Windows has issues with spaces and paths -The path will need to + be set correctly.) + The default ctest configuration is defined as "Release". It can be changed + with the CTEST_CONFIGURATION_TYPE script option. Note that this must + be the same as the value used with the -C command line option. + On Windows, you can set the CTEST_VSVERS script option to either + 64_VS2022 or 64_VS2019. Alternately, you can set the script + CTEST_CMAKE_GENERATOR option to "Visual Studio 16 2019" or "Visual Studio 17 2022", + and the CMAKE_GENERATOR_ARCHITECTURE script option to "x64". + + The default build configuration is defined to build and use static libraries. + + Shared libraries and other options can be changed by editing the + HDF5PLoptions.cmake file. + + If the defaults are okay, execute from this directory: + ctest -S HDF5PL_Examples.cmake -C Release -VV -O test.log + If the defaults need change, execute from this directory: + ctest -S HDF5PL_Examples.cmake,CTEST_SOURCE_NAME=MyExamples,INSTALLDIR=MyLocation -C Release -VV -O test.log + + When executed, the ctest script will save the results to the log file, test.log, as + indicated by the ctest command. If you wish to see more build and test information, + add "-VV" to the ctest command. The output should show; + 100% tests passed, 0 tests failed out of 206. + + 3. Using Command Line CMake + A. Visual Configuration + The visual CMake executable is named "cmake-gui.exe" on Windows and should be + available in your Start menu. For Linux, UNIX, and Mac users the + executable is named "cmake-gui" or the ncurses-based "ccmake" and can be found + where CMake was installed. + + Specify the source and build directories. Make the build and source + directories different. For example on Windows, if the source is at + C:\MyHDFstuff\hdf5pl, then use C:\MyHDFstuff\hdf5pl\build or + C:\MyHDFstuff\build\hdf5pl for the build directory. + + Click the Configure button. If this is the first time you are + running cmake-gui in this directory, you will be prompted for the + generator you wish to use (for example on Windows, Visual Studio 15). + CMake will read in the CMakeLists.txt files from the source directory and + display options for the HDF5PL Examples project. After the first configure you + can adjust the cache settings and/or specify locations of other programs. + + Any conflicts or new values will be highlighted by the configure + process in red. Once you are happy with all the settings and there are no + more values in red, click the Generate button to produce the appropriate + build files. + + On Windows, if you are using a Visual Studio generator, the solution and + project files will be created in the build folder. + + On linux, if you are using the Unix Makefiles generator, the Makefiles will + be created in the build folder. + + B. Alternative Command Line Configuration + Users can perform the configuration step without using the visual + cmake-gui program. The following is an example command line + configuration step executed within the build directory: + + cmake -G "" [-D] + + Where is + * MSYS Makefiles + * MinGW Makefiles + * NMake Makefiles + * Unix Makefiles + * Visual Studio 15 2017 + * Visual Studio 15 2017 Win64 + * Visual Studio 16 2019 + * ... in addition VS2019 will need to set the "-A" option, + * ... [Win32, x64, ARM, ARM64] + * Visual Studio 17 2022 + * ... in addition VS2022 will need to set the "-A" option, + * ... [Win32, x64, ARM, ARM64] + + is: + * H5PL_BUILD_TESTING:BOOL=ON + + Example command line on Windows in c:\MyHDFstuff\hdf5pl\build directory: + + cmake -G "Visual Studio 17 2022" -DH5PL_BUILD_TESTING:BOOL=ON .. + + C. Build HDF5 Plugin examples + + On Windows, you can build HDF5 Plugin applications using either the Visual Studio Environment + or the command line. The command line is normally used on linux, Unix, and Mac. + + To build from the command line, navigate to your build directory and + execute the following: + + cmake --build . --config {Debug | Release} + + NOTE: "--config {Debug | Release}" may be optional on your platform. We + recommend choosing either Debug or Release on Windows. If you are + using the pre-built binaries from HDF, use Release. + + If you wish to use the Visual Studio environment, open the solution + file in your build directory. Be sure to select either Debug or + Release and build the solution. + + D. Test HDF5 Plugin Examples. + + To test the build, navigate to your build directory and execute: + + ctest . -C {Debug | Release} + + NOTE: "-C {Debug | Release}" may be optional on your platform. We + recommend choosing either Debug or Release to match the build + step on Windows. + + 4. The files that support building with CMake are all of the files in the + config/cmake folder, the CMakeLists.txt files in each source folder. + + CTestConfig.cmake is specific to the internal testing + performed by The HDF Group. It should be altered for the user's + installation and needs. The cacheinit.cmake file settings are used by + The HDF Group for daily testing. It should be altered/ignored for the user's + installation and needs. + + +======================================================================== +III. Defaults in the CMakePresets.json file +======================================================================== + +#### DEFAULT: +#### "generator": "Ninja" +#### "binaryDir": "${sourceParentDir}/build/${presetName}" +#### "name": "ci-StdShar" +#### "CMAKE_BUILD_TYPE": "RelWithDebInfo" +#### "HDF5_NAMESPACE": {"type": "STRING", "value": "hdf5::"}, +#### "HDF5_PACKAGE_NAME": {"type": "STRING", "value": "hdf5"}, +#### "H5PL_BUILD_TESTING": "ON" + + +======================================================================== +IV. Defaults in the HDF5_Examples_options.cmake file +======================================================================== + +#### DEFAULT: +#### H5PL_BUILD_TESTING:BOOL=OFF + + +************************************************************************ + +Need further assistance, send email to help@hdfgroup.org + diff --git a/SZ3/config/cmake/binex/config/cmake-presets/hidden-presets.json b/SZ3/config/cmake/binex/config/cmake-presets/hidden-presets.json new file mode 100644 index 00000000..c78b4c4e --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake-presets/hidden-presets.json @@ -0,0 +1,514 @@ +{ + "version": 6, + "configurePresets": [ + { + "name": "ci-base", + "displayName": "Basic Config", + "description": "Basic build using Ninja generator", + "generator": "Ninja", + "hidden": true, + "binaryDir": "${sourceParentDir}/build/${presetName}", + "installDir": "${sourceParentDir}/install/${presetName}" + }, + { + "name": "ci-x64", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-x86", + "architecture": { + "value": "Win32", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-arm64", + "architecture": { + "value": "ARM64", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-Debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "hidden": true + }, + { + "name": "ci-Release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + }, + "hidden": true + }, + { + "name": "ci-MSVC", + "hidden": true, + "cacheVariables": { + "CMAKE_C_COMPILER": "cl", + "CMAKE_CXX_COMPILER": "cl" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "ci-macos-arm64", + "hidden": true, + "cacheVariables": { + "CMAKE_OSX_ARCHITECTURES": "arm64" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + } + }, + { + "name": "ci-macos-x86_64", + "hidden": true, + "cacheVariables": { + "CMAKE_OSX_ARCHITECTURES": "x86_64" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Darwin" + } + }, + { + "name": "ci-Clang", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/clang.cmake" + } + }, + { + "name": "ci-GNUC", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/gcc.cmake" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Linux" + } + }, + { + "name": "ci-Intel", + "hidden": true + }, + { + "name": "ci-x64-Debug-MSVC", + "description": "MSVC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-MSVC" + ] + }, + { + "name": "ci-x64-Release-MSVC", + "description": "MSVC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-MSVC" + ] + }, + { + "name": "ci-x64-Debug-Clang", + "description": "Clang/LLVM for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-Clang" + ] + }, + { + "name": "ci-x64-Release-Clang", + "description": "Clang/LLVM for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-Clang" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "description": "GNUC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-GNUC" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "description": "GNUC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-GNUC" + ] + }, + { + "name": "ci-macos-arm64-Debug-Clang", + "description": "Clang/LLVM for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Debug", + "ci-Clang" + ] + }, + { + "name": "ci-macos-arm64-Release-Clang", + "description": "Clang/LLVM for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Release", + "ci-Clang" + ] + }, + { + "name": "ci-macos-arm64-Debug-GNUC", + "description": "GNUC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Debug", + "ci-GNUC" + ] + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "description": "GNUC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-macos-arm64", + "ci-Release", + "ci-GNUC" + ] + }, + { + "name": "ci-x64-Debug-Intel", + "description": "Intel for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-Intel" + ] + }, + { + "name": "ci-x64-Release-Intel", + "description": "Intel for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-Intel" + ] + } + ], + "buildPresets": [ + { + "name": "ci-base", + "configurePreset": "ci-base", + "hidden": true, + "verbose": true, + "jobs": 8 + }, + { + "name": "ci-x64-Debug-MSVC", + "configurePreset": "ci-x64-Debug-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "Debug" + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "RelWithDebInfo" + }, + { + "name": "ci-x64-Debug-Clang", + "configurePreset": "ci-x64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "configurePreset": "ci-x64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Debug-Clang", + "configurePreset": "ci-macos-arm64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-Clang", + "configurePreset": "ci-macos-arm64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Debug-GNUC", + "configurePreset": "ci-macos-arm64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "configurePreset": "ci-macos-arm64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-Intel", + "configurePreset": "ci-x64-Debug-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "Debug" + }, + { + "name": "ci-x64-Release-Intel", + "configurePreset": "ci-x64-Release-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "RelWithDebInfo" + } + ], + "testPresets": [ + { + "name": "ci-base", + "configurePreset": "ci-base", + "output": { + "outputOnFailure": false, + "shortProgress": true, + "verbosity": "verbose" + }, + "hidden": true, + "execution": { + "noTestsAction": "error", + "timeout": 600, + "jobs": 8 + } + }, + { + "name": "ci-x64-Debug-MSVC", + "configurePreset": "ci-x64-Debug-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "Debug" + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ], + "configuration": "RelWithDebInfo" + }, + { + "name": "ci-x64-Debug-Clang", + "configurePreset": "ci-x64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "configurePreset": "ci-x64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-Debug-Clang", + "configurePreset": "ci-macos-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-Clang", + "configurePreset": "ci-macos-arm64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Debug-GNUC", + "configurePreset": "ci-macos-arm64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "configurePreset": "ci-macos-arm64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-Intel", + "configurePreset": "ci-x64-Debug-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Intel", + "configurePreset": "ci-x64-Release-Intel", + "hidden": true, + "inherits": [ + "ci-base" + ] + } + ], + "packagePresets": [ + { + "name": "ci-base", + "hidden": true, + "output": { + "verbose": true + } + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": "ci-base", + "configurations": ["RelWithDebInfo"] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-macos-arm64-Release-Clang", + "configurePreset": "ci-macos-arm64-Release-Clang", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-macos-arm64-Release-GNUC", + "configurePreset": "ci-macos-arm64-Release-GNUC", + "hidden": true, + "inherits": "ci-base" + }, + { + "name": "ci-x64-Release-Intel", + "configurePreset": "ci-x64-Release-Intel", + "hidden": true, + "inherits": "ci-base" + } + ] +} diff --git a/SZ3/config/cmake/binex/config/cmake/CTestCustom.cmake b/SZ3/config/cmake/binex/config/cmake/CTestCustom.cmake new file mode 100755 index 00000000..77cea8fb --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake/CTestCustom.cmake @@ -0,0 +1,22 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +set (CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 1500) + +set (CTEST_CUSTOM_WARNING_EXCEPTION + ${CTEST_CUSTOM_WARNING_EXCEPTION} + "POSIX name for this item is deprecated" + "disabling jobserver mode" +) + +set (CTEST_CUSTOM_MEMCHECK_IGNORE + ${CTEST_CUSTOM_MEMCHECK_IGNORE} +) diff --git a/SZ3/config/cmake/binex/config/cmake/HDFMacros.cmake b/SZ3/config/cmake/binex/config/cmake/HDFMacros.cmake new file mode 100644 index 00000000..60e69e7d --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake/HDFMacros.cmake @@ -0,0 +1,519 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +#------------------------------------------------------------------------------- +# Setup output Directories +#----------------------------------------------------------------------------- +macro (SET_HDF_OUTPUT_DIRS package_prefix) + if (NOT ${package_prefix}_EXTERNALLY_CONFIGURED) + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." + ) + set (CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" + ) + set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." + ) + set (CMAKE_Fortran_MODULE_DIRECTORY + ${PROJECT_BINARY_DIR}/mod CACHE PATH "Single Directory for all fortran modules." + ) + get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if(_isMultiConfig) + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}) + set (CMAKE_PDB_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all pdb files." + ) + else () + set (CMAKE_TEST_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + endif () + else () + # if we are externally configured, but the project uses old cmake scripts + # this may not be set and utilities like H5detect will fail + if (NOT CMAKE_RUNTIME_OUTPUT_DIRECTORY) + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${EXECUTABLE_OUTPUT_PATH}) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (SET_HDF_BUILD_TYPE) + get_property (_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) + if (_isMultiConfig) + # HDF_CFG_BUILD_TYPE is used in the Fortran install commands for the build location of the .mod files + set (HDF_CFG_BUILD_TYPE \${CMAKE_INSTALL_CONFIG_NAME}) + if (CMAKE_BUILD_TYPE) + # set the default to the specified command line define + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + # set the default to the MultiConfig variable + set (HDF_CFG_NAME "$") + endif () + else () + set (HDF_CFG_BUILD_TYPE ".") + if (CMAKE_BUILD_TYPE) + set (HDF_CFG_NAME ${CMAKE_BUILD_TYPE}) + else () + set (HDF_CFG_NAME "Release") + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (SET_GLOBAL_VARIABLE name value) + set (${name} ${value} CACHE INTERNAL "Used to pass variables between directories" FORCE) +endmacro () + +#------------------------------------------------------------------------------- +macro (IDE_GENERATED_PROPERTIES SOURCE_PATH HEADERS SOURCES) + #set (source_group_path "Source/AIM/${NAME}") + string (REPLACE "/" "\\\\" source_group_path ${SOURCE_PATH}) + source_group (${source_group_path} FILES ${HEADERS} ${SOURCES}) + + #-- The following is needed if we ever start to use OS X Frameworks but only + #-- works on CMake 2.6 and greater + #set_property (SOURCE ${HEADERS} + # PROPERTY MACOSX_PACKAGE_LOCATION Headers/${NAME} + #) +endmacro () + +#------------------------------------------------------------------------------- +macro (IDE_SOURCE_PROPERTIES SOURCE_PATH HEADERS SOURCES) + # install (FILES ${HEADERS} + # DESTINATION include/R3D/${NAME} + # COMPONENT Headers + # ) + + string (REPLACE "/" "\\\\" source_group_path ${SOURCE_PATH} ) + source_group (${source_group_path} FILES ${HEADERS} ${SOURCES}) + + #-- The following is needed if we ever start to use OS X Frameworks but only + #-- works on CMake 2.6 and greater + #set_property (SOURCE ${HEADERS} + # PROPERTY MACOSX_PACKAGE_LOCATION Headers/${NAME} + #) +endmacro () + +#------------------------------------------------------------------------------- +macro (INSTALL_TARGET_PDB libtarget targetdestination targetcomponent) + option (DISABLE_PDB_FILES "Do not install PDB files" OFF) + mark_as_advanced (DISABLE_PDB_FILES) + if (WIN32 AND MSVC AND NOT DISABLE_PDB_FILES) + get_target_property (target_type ${libtarget} TYPE) + if (${target_type} MATCHES "SHARED") + set (targetfilename $) + else () + get_property (target_name TARGET ${libtarget} PROPERTY "$,$>,OUTPUT_NAME_DEBUG,OUTPUT_NAME_RELWITHDEBINFO>") + set (targetfilename $/${target_name}.pdb) + endif () + install ( + FILES ${targetfilename} + DESTINATION ${targetdestination} + CONFIGURATIONS Debug RelWithDebInfo + COMPONENT ${targetcomponent} + OPTIONAL + ) + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (INSTALL_PROGRAM_PDB progtarget targetdestination targetcomponent) + if (WIN32 AND MSVC) + install ( + FILES $ + DESTINATION ${targetdestination} + CONFIGURATIONS Debug RelWithDebInfo + COMPONENT ${targetcomponent} + OPTIONAL + ) + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF_SET_BASE_OPTIONS libtarget libname libtype) + if (${libtype} MATCHES "SHARED") + set (LIB_RELEASE_NAME "${libname}") + set (LIB_DEBUG_NAME "${libname}${CMAKE_DEBUG_POSTFIX}") + else () + if (WIN32 AND NOT MINGW) + if ("${ARGN}" STREQUAL "NOPREFIX") + set (LIB_RELEASE_NAME "${libname}") + set (LIB_DEBUG_NAME "${libname}${CMAKE_DEBUG_POSTFIX}") + else () + set (LIB_RELEASE_NAME "lib${libname}") + set (LIB_DEBUG_NAME "lib${libname}${CMAKE_DEBUG_POSTFIX}") + endif () + else () + set (LIB_RELEASE_NAME "${libname}") + set (LIB_DEBUG_NAME "${libname}${CMAKE_DEBUG_POSTFIX}") + endif () + endif () + + set_target_properties (${libtarget} PROPERTIES + OUTPUT_NAME ${LIB_RELEASE_NAME} +# OUTPUT_NAME_DEBUG ${LIB_DEBUG_NAME} + OUTPUT_NAME_RELEASE ${LIB_RELEASE_NAME} + OUTPUT_NAME_MINSIZEREL ${LIB_RELEASE_NAME} + OUTPUT_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} + ) + + if (${libtype} MATCHES "STATIC") + if (WIN32) + set_target_properties (${libtarget} PROPERTIES + COMPILE_PDB_NAME_DEBUG ${LIB_DEBUG_NAME} + COMPILE_PDB_NAME_RELEASE ${LIB_RELEASE_NAME} + COMPILE_PDB_NAME_MINSIZEREL ${LIB_RELEASE_NAME} + COMPILE_PDB_NAME_RELWITHDEBINFO ${LIB_RELEASE_NAME} + COMPILE_PDB_OUTPUT_DIRECTORY "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}" + ) + endif () + endif () + + option (HDF_MSVC_NAMING_CONVENTION "Use MSVC Naming conventions for Shared Libraries" OFF) + if (HDF_MSVC_NAMING_CONVENTION AND MINGW AND ${libtype} MATCHES "SHARED") + set_target_properties (${libtarget} PROPERTIES + IMPORT_SUFFIX ".lib" + IMPORT_PREFIX "" + PREFIX "" + ) + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF_SET_LIB_VERSIONS pkg_name libtarget defaultlibname libtype) + set (libname "${defaultlibname}") + HDF_SET_BASE_OPTIONS (${libtarget} ${libname} ${libtype} ${ARGN}) + + if (${libtype} MATCHES "SHARED") + set (LIB_PACKAGE_SOVERSION ${${pkg_name}_VERS_MAJOR}) + if (WIN32) + set (LIB_VERSION ${${pkg_name}_PACKAGE_VERSION_MAJOR}) + else () + set (LIB_VERSION ${${pkg_name}_PACKAGE_VERSION}) + endif () + set_target_properties (${libtarget} PROPERTIES VERSION ${LIB_VERSION}) + if (WIN32) + set (${libname} "${libname}-${LIB_PACKAGE_SOVERSION}") + else () + set_target_properties (${libtarget} PROPERTIES SOVERSION ${LIB_PACKAGE_SOVERSION}) + endif () + endif () + + #-- Apple Specific install_name for libraries + if (APPLE) + option (${pkg_name}_BUILD_WITH_INSTALL_NAME "Build with library install_name set to the installation path" OFF) + if (${pkg_name}_BUILD_WITH_INSTALL_NAME) + set_target_properties(${libtarget} PROPERTIES + LINK_FLAGS "-current_version ${${pkg_name}_PACKAGE_VERSION} -compatibility_version ${${pkg_name}_PACKAGE_VERSION}" + INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib" + BUILD_WITH_INSTALL_RPATH ${${pkg_name}_BUILD_WITH_INSTALL_NAME} + ) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (HDF_IMPORT_SET_LIB_OPTIONS libtarget libname libtype libversion) + HDF_SET_BASE_OPTIONS (${libtarget} ${libname} ${libtype} ${ARGN}) + + if (${importtype} MATCHES "IMPORT") + set (importprefix "${CMAKE_STATIC_LIBRARY_PREFIX}") + endif () + if (${HDF_CFG_NAME} MATCHES "Debug") + set (IMPORT_LIB_NAME ${LIB_DEBUG_NAME}) + else () + set (IMPORT_LIB_NAME ${LIB_RELEASE_NAME}) + endif () + + if (${libtype} MATCHES "SHARED") + if (WIN32) + if (MINGW) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}.lib" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + else () + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + endif () + else () + if (MINGW) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}.lib" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + elseif (CYGWIN) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_IMPLIB "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_IMPORT_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + else () + set_target_properties (${libtarget} PROPERTIES + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}" + IMPORTED_SONAME "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_SHARED_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${libversion}" + SOVERSION "${libversion}" + ) + endif () + endif () + else () + if (WIN32 AND NOT MINGW) + set_target_properties (${libtarget} PROPERTIES + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${HDF_CFG_NAME}/${IMPORT_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + ) + else () + set_target_properties (${libtarget} PROPERTIES + IMPORTED_LOCATION "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/${CMAKE_STATIC_LIBRARY_PREFIX}${IMPORT_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + ) + endif () + endif () +endmacro () + +#------------------------------------------------------------------------------- +macro (TARGET_C_PROPERTIES wintarget libtype) + target_compile_options(${wintarget} PRIVATE + "$<$:${WIN_COMPILE_FLAGS}>" + "$<$:${WIN_COMPILE_FLAGS}>" + ) + if(MSVC) + set_property(TARGET ${wintarget} APPEND PROPERTY LINK_FLAGS "${WIN_LINK_FLAGS}") + endif() +endmacro () + +#----------------------------------------------------------------------------- +# Configure the README.txt file for the binary package +#----------------------------------------------------------------------------- +macro (PLUGIN_README_PROPERTIES pkg_name) + set (BINARY_SYSTEM_NAME ${CMAKE_SYSTEM_NAME}) + set (BINARY_PLATFORM "${CMAKE_SYSTEM_NAME}") + if (WIN32) + set (BINARY_EXAMPLE_ENDING "zip") + set (BINARY_INSTALL_ENDING "msi") + set (BINARY_COMPRESS_ENDING "zip") + if (CMAKE_CL_64) + set (BINARY_SYSTEM_NAME "win64") + else () + set (BINARY_SYSTEM_NAME "win32") + endif () + if (${CMAKE_SYSTEM_VERSION} MATCHES "6.1") + set (BINARY_PLATFORM "${BINARY_PLATFORM} 7") + elseif (${CMAKE_SYSTEM_VERSION} MATCHES "6.2") + set (BINARY_PLATFORM "${BINARY_PLATFORM} 8") + elseif (${CMAKE_SYSTEM_VERSION} MATCHES "6.3") + set (BINARY_PLATFORM "${BINARY_PLATFORM} 10") + endif () + if (CMAKE_C_COMPILER_ID MATCHES "Intel") + set (BINARY_PLATFORM "${BINARY_PLATFORM} Intel") + if (${CMAKE_C_COMPILER_VERSION} MATCHES "^17.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel 17") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^18.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel 18") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel 19") + else () + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using Intel ${CMAKE_C_COMPILER_VERSION}") + endif () + elseif (CMAKE_C_COMPILER_ID MATCHES "MSVC") + set (BINARY_PLATFORM "${BINARY_PLATFORM} ${MSVC_C_ARCHITECTURE_ID}") + if (${CMAKE_C_COMPILER_VERSION} MATCHES "^16.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2010") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^15.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2008") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^17.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2012") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^18.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2013") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.*") + if (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.0.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2015") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.1.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2017") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.2.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2019") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.3.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2022") + elseif (${CMAKE_C_COMPILER_VERSION} MATCHES "^19.4.*") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO 2022") + else () + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO ????") + endif () + else () + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using VISUAL STUDIO ${CMAKE_C_COMPILER_VERSION}") + endif () + endif () + elseif (APPLE) + set (BINARY_EXAMPLE_ENDING "tar.gz") + set (BINARY_COMPRESS_ENDING "tar.gz") + set (BINARY_INSTALL_ENDING "sh") # if packaging changes - use dmg + set (BINARY_PLATFORM "${BINARY_PLATFORM} ${CMAKE_SYSTEM_VERSION} ${CMAKE_SYSTEM_PROCESSOR}") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using ${CMAKE_C_COMPILER_ID} C ${CMAKE_C_COMPILER_VERSION}") + else () + set (BINARY_EXAMPLE_ENDING "tar.gz") + set (BINARY_COMPRESS_ENDING "tar.gz") + set (BINARY_INSTALL_ENDING "sh/deb/rpm") + set (BINARY_PLATFORM "${BINARY_PLATFORM} ${CMAKE_SYSTEM_VERSION} ${CMAKE_SYSTEM_PROCESSOR}") + set (BINARY_PLATFORM "${BINARY_PLATFORM}, using ${CMAKE_C_COMPILER_ID} C ${CMAKE_C_COMPILER_VERSION}") + endif () + + configure_file ( + ${${pkg_name}_RESOURCES_DIR}/README.txt.cmake.in + ${CMAKE_BINARY_DIR}/README.txt @ONLY + ) +endmacro () + +macro (HDFTEST_COPY_FILE src dest target) + add_custom_command( + OUTPUT "${dest}" + COMMAND "${CMAKE_COMMAND}" + ARGS -E copy_if_different "${src}" "${dest}" + DEPENDS "${src}" + ) + list (APPEND ${target}_list "${dest}") +endmacro () + +macro (HDF_DIR_PATHS package_prefix) + option (H5PL_USE_GNU_DIRS "ON to use GNU Coding Standard install directory variables, OFF to use historical settings" OFF) + if (H5PL_USE_GNU_DIRS) + include(GNUInstallDirs) + if (NOT ${package_prefix}_INSTALL_BIN_DIR) + set (${package_prefix}_INSTALL_BIN_DIR ${CMAKE_INSTALL_BINDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_LIB_DIR) + set (${package_prefix}_INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_JAR_DIR) + set (${package_prefix}_INSTALL_JAR_DIR ${CMAKE_INSTALL_LIBDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_INCLUDE_DIR) + set (${package_prefix}_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}) + endif () + if (NOT ${package_prefix}_INSTALL_MODULE_DIR) + set (${package_prefix}_INSTALL_MODULE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/mod) + endif () + if (NOT ${package_prefix}_INSTALL_DATA_DIR) + set (${package_prefix}_INSTALL_DATA_DIR ${CMAKE_INSTALL_DATADIR}) + endif () + if (NOT ${package_prefix}_INSTALL_CMAKE_DIR) + set (${package_prefix}_INSTALL_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake) + endif () + if (NOT ${package_prefix}_INSTALL_DOC_DIR) + set (${package_prefix}_INSTALL_DOC_DIR ${CMAKE_INSTALL_DOCDIR}) + endif () + message(STATUS "GNU: ${${package_prefix}_INSTALL_DOC_DIR}") + endif () + + if (APPLE) + option (${package_prefix}_BUILD_FRAMEWORKS "ON to build as frameworks libraries, OFF to build according to BUILD_SHARED_LIBS" OFF) + endif () + + if (NOT ${package_prefix}_INSTALL_BIN_DIR) + set (${package_prefix}_INSTALL_BIN_DIR bin) + endif () + if (NOT ${package_prefix}_INSTALL_LIB_DIR) + if (APPLE) + if (${package_prefix}_BUILD_FRAMEWORKS) + set (${package_prefix}_INSTALL_JAR_DIR ../Java) + else () + set (${package_prefix}_INSTALL_JAR_DIR lib) + endif () + set (${package_prefix}_INSTALL_FMWK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + else () + set (${package_prefix}_INSTALL_JAR_DIR lib) + endif () + set (${package_prefix}_INSTALL_LIB_DIR lib) + endif () + if (NOT ${package_prefix}_INSTALL_INCLUDE_DIR) + set (${package_prefix}_INSTALL_INCLUDE_DIR include) + endif () + if (NOT ${package_prefix}_INSTALL_MODULE_DIR) + set (${package_prefix}_INSTALL_MODULE_DIR mod) + endif () + if (NOT ${package_prefix}_INSTALL_DATA_DIR) + if (NOT MSVC) + if (APPLE) + if (${package_prefix}_BUILD_FRAMEWORKS) + set (${package_prefix}_INSTALL_EXTRA_DIR ../SharedSupport) + else () + set (${package_prefix}_INSTALL_EXTRA_DIR share) + endif () + set (${package_prefix}_INSTALL_FWRK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + endif () + set (${package_prefix}_INSTALL_DATA_DIR share) + else () + set (${package_prefix}_INSTALL_DATA_DIR ".") + endif () + endif () + if (NOT ${package_prefix}_INSTALL_CMAKE_DIR) + set (${package_prefix}_INSTALL_CMAKE_DIR cmake) + endif () + if (NOT ${package_prefix}_INSTALL_DOC_DIR) + set (${package_prefix}_INSTALL_DOC_DIR ${${package_prefix}_INSTALL_DATA_DIR}) + endif () + message(VERBOSE "Final: ${${package_prefix}_INSTALL_DOC_DIR}") + + # Always use full RPATH, i.e. don't skip the full RPATH for the build tree + set (CMAKE_SKIP_BUILD_RPATH OFF) + # when building, don't use the install RPATH already + # (but later on when installing) + set (CMAKE_INSTALL_RPATH_USE_LINK_PATH OFF) + # add the automatically determined parts of the RPATH + # which point to directories outside the build tree to the install RPATH + set (CMAKE_BUILD_WITH_INSTALL_RPATH ON) + if (APPLE) + set (CMAKE_INSTALL_NAME_DIR "@rpath") + set (CMAKE_INSTALL_RPATH + "@executable_path/../${${package_prefix}_INSTALL_LIB_DIR}" + "@executable_path/" + "@loader_path/../${${package_prefix}_INSTALL_LIB_DIR}" + "@loader_path/" + ) + else () + set (CMAKE_INSTALL_RPATH "\$ORIGIN/../${${package_prefix}_INSTALL_LIB_DIR}:\$ORIGIN/") + endif () + + if (DEFINED ADDITIONAL_CMAKE_PREFIX_PATH AND EXISTS "${ADDITIONAL_CMAKE_PREFIX_PATH}") + set (CMAKE_PREFIX_PATH ${ADDITIONAL_CMAKE_PREFIX_PATH} ${CMAKE_PREFIX_PATH}) + endif () + + #set the default debug suffix for all library targets + if(NOT CMAKE_DEBUG_POSTFIX) + if (WIN32) + set (CMAKE_DEBUG_POSTFIX "_D") + else () + set (CMAKE_DEBUG_POSTFIX "_debug") + endif () + endif () + + SET_HDF_BUILD_TYPE() + + SET_HDF_OUTPUT_DIRS(${package_prefix}) + + include (FetchContent) +endmacro () + +macro (ADD_H5_FLAGS h5_flag_var infile) + file (STRINGS ${infile} TEST_FLAG_STREAM) + list (LENGTH TEST_FLAG_STREAM len_flag) + if (len_flag GREATER 0) + math (EXPR _FP_LEN "${len_flag} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + list (GET TEST_FLAG_STREAM ${line} str_flag) + string (REGEX REPLACE "^#.*" "" str_flag "${str_flag}") + if (str_flag) + list (APPEND ${h5_flag_var} "${str_flag}") + endif () + endforeach () + endif () +endmacro () diff --git a/SZ3/config/cmake/binex/config/cmake/HDFPluginMacros.cmake b/SZ3/config/cmake/binex/config/cmake/HDFPluginMacros.cmake new file mode 100644 index 00000000..9131edc5 --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake/HDFPluginMacros.cmake @@ -0,0 +1,279 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +#------------------------------------------------------------------------------- +macro (BASIC_SETTINGS varname) + #----------------------------------------------------------------------------- + # Instructions for use : Normal Build + # + # For standard build of HDF5 plugin examples. + #----------------------------------------------------------------------------- + set (CMAKE_NO_SYSTEM_FROM_IMPORTED 1) + + #----------------------------------------------------------------------------- + # Allow Visual Studio solution directories + #----------------------------------------------------------------------------- + # Provide a way for Visual Studio Express users to turn OFF the new FOLDER + # organization feature. Default to ON for non-Express users. Express users must + # explicitly turn off this option to build HDF5 in the Express IDE... + # + option (HDFPL_USE_FOLDERS "Enable folder grouping of projects in IDEs." ON) + mark_as_advanced (HDFPL_USE_FOLDERS) + if (HDFPL_USE_FOLDERS) + set_property (GLOBAL PROPERTY USE_FOLDERS ON) + endif () + + #----------------------------------------------------------------------------- + # Define some CMake variables for use later in the project + #----------------------------------------------------------------------------- + set (H5PL_RESOURCES_DIR ${PROJECT_SOURCE_DIR}/config/cmake) + set (HDF5PL_SRC_DIR ${PROJECT_SOURCE_DIR}/example) + + #----------------------------------------------------------------------------- + # Include some macros for reusable code + #----------------------------------------------------------------------------- + include (${H5PL_RESOURCES_DIR}/HDFMacros.cmake) + + SET_HDF_BUILD_TYPE() + + #----------------------------------------------------------------------------- + # Setup output Directories + #----------------------------------------------------------------------------- + set (CMAKE_RUNTIME_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Executables." + ) + set (CMAKE_LIBRARY_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all Libraries" + ) + set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY + ${PROJECT_BINARY_DIR}/bin CACHE PATH "Single Directory for all static libraries." + ) + + #----------------------------------------------------------------------------- + # When building utility executables that generate other (source) files : + # we make use of the following variables defined in the root CMakeLists. + # Certain systems may add /Debug or /Release to output paths + # and we need to call the executable from inside the CMake configuration + #----------------------------------------------------------------------------- + set (EXE_EXT "") + if (WIN32) + set (EXE_EXT ".exe") + add_definitions (-D_BIND_TO_CURRENT_VCLIBS_VERSION=1) + add_definitions (-D_CRT_SECURE_NO_WARNINGS) + add_definitions (-D_CONSOLE) + endif () + + #----------------------------------------------------------------------------- + # Plugins are MODULEs that must be built as Shared libs + #----------------------------------------------------------------------------- + set (BUILD_SHARED_LIBS ON CACHE BOOL "Build Shared Libraries" FORCE) + set (LIB_TYPE SHARED) + set (CMAKE_POSITION_INDEPENDENT_CODE ON) + + if (MSVC) + set (CMAKE_MFC_FLAG 0) + endif () + + set (CMAKE_C_STANDARD 99) + set (CMAKE_C_STANDARD_REQUIRED TRUE) + + if (HDF5_BUILD_CPP_LIB) + ENABLE_LANGUAGE (CXX) + + set (CMAKE_CXX_STANDARD 98) + set (CMAKE_CXX_STANDARD_REQUIRED TRUE) + set (CMAKE_CXX_EXTENSIONS OFF) + endif () + + #----------------------------------------------------------------------------- + # Compiler specific flags : Shouldn't there be compiler tests for these + #----------------------------------------------------------------------------- + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS} -std=c99 -fomit-frame-pointer -finline-functions -fno-common") + endif () + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS} -fomit-frame-pointer -finline-functions -fno-common") + endif () + + #----------------------------------------------------------------------------- + # Option to allow the user to disable compiler warnings + #----------------------------------------------------------------------------- + option (HDF_DISABLE_COMPILER_WARNINGS "Disable compiler warnings" OFF) + if (HDF_DISABLE_COMPILER_WARNINGS) + # MSVC uses /w to suppress warnings. It also complains if another + # warning level is given, so remove it. + if (MSVC) + set (HDF_WARNINGS_BLOCKED 1) + string (REGEX REPLACE "(^| )([/-])W[0-9]( |$)" " " CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /w") + string (REGEX REPLACE "(^| )([/-])W[0-9]( |$)" " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /w") + endif () + if (WIN32) + add_definitions (-D_CRT_SECURE_NO_WARNINGS) + endif () + # Borland uses -w- to suppress warnings. + if (BORLAND) + set (HDF_WARNINGS_BLOCKED 1) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w-") + endif () + + # Most compilers use -w to suppress warnings. + if (NOT HDF_WARNINGS_BLOCKED) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") + endif () + endif () + + #----------------------------------------------------------------------------- + # This is in here to help some of the GCC based IDES like Eclipse + # and code blocks parse the compiler errors and warnings better. + #----------------------------------------------------------------------------- + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fmessage-length=0") + endif () + if (CMAKE_CXX_COMPILER_LOADED AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0") + endif () + + #----------------------------------------------------------------------------- + # All examples need the main include directories + #----------------------------------------------------------------------------- + INCLUDE_DIRECTORIES (${PROJECT_BINARY_DIR}) +endmacro () + +macro (HDF5_SUPPORT link_hdf) + set (CMAKE_MODULE_PATH ${H5PL_RESOURCES_DIR} ${CMAKE_MODULE_PATH}) + set (CMAKE_POSITION_INDEPENDENT_CODE ON) + # plugins must use SHARED libraries + option (USE_SHARED_LIBS "Use Shared Libraries" ON) + + if (NOT H5PL_HDF5_HEADER) + if (USE_SHARED_LIBS) + set (FIND_HDF_COMPONENTS C shared) + else () + set (FIND_HDF_COMPONENTS C static) + endif () + message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") + set (SEARCH_PACKAGE_NAME ${HDF5_PACKAGE_NAME}) + + find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) + message (STATUS "HDF5 C libs:${HDF5_FOUND} static:${HDF5_static_C_FOUND} and shared:${HDF5_shared_C_FOUND}") + if (HDF5_FOUND) + if (NOT HDF5_static_C_FOUND AND NOT HDF5_shared_C_FOUND) + #find library from non-dual-binary package + set (FIND_HDF_COMPONENTS C) + message (STATUS "HDF5 find comps: ${FIND_HDF_COMPONENTS}") + + find_package (HDF5 NAMES ${SEARCH_PACKAGE_NAME} COMPONENTS ${FIND_HDF_COMPONENTS}) + message (STATUS "HDF5 libs:${HDF5_FOUND} C:${HDF5_C_FOUND}") + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_LIBRARIES}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_LIBRARIES}) + if (HDF5_BUILD_SHARED_LIBS) + add_definitions (-DH5_BUILT_AS_DYNAMIC_LIB) + else () + add_definitions (-DH5_BUILT_AS_STATIC_LIB) + endif () + endif () + if (USE_SHARED_LIBS AND WIN32) + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dumpdll") + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repackdll") + else () + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump") + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack") + endif () + set (H5PL_HDF5_DUMP_EXECUTABLE $) + set (H5PL_HDF5_REPACK_EXECUTABLE $) + else () + get_filename_component (_LIBRARY_PATH ${HDF5_INCLUDE_DIR} DIRECTORY) + set (HDF5_LIBRARY_PATH "${_LIBRARY_PATH}/lib") + if (USE_SHARED_LIBS AND HDF5_shared_C_FOUND) + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_C_SHARED_LIBRARY}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_C_SHARED_LIBRARY}) + endif () + else () + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_C_STATIC_LIBRARY}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_C_STATIC_LIBRARY}) + endif () + endif () + if (HDF5_VERSION VERSION_LESS "1.14.4" AND NOT HDF5_shared_C_FOUND) + if (NOT TARGET ${HDF5_NAMESPACE}h5dump-shared) + add_executable (${HDF5_NAMESPACE}h5dump-shared IMPORTED) + endif () + if (NOT TARGET ${HDF5_NAMESPACE}h5repack-shared) + add_executable (${HDF5_NAMESPACE}h5repack-shared IMPORTED) + endif () + set (H5PL_HDF5_DUMP_EXECUTABLE $) + set (H5PL_HDF5_REPACK_EXECUTABLE $) + else () + if (NOT TARGET ${HDF5_NAMESPACE}h5dump) + add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) + endif() + if (NOT TARGET ${HDF5_NAMESPACE}h5repack) + add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) + endif() + set (H5PL_HDF5_DUMP_EXECUTABLE $) + set (H5PL_HDF5_REPACK_EXECUTABLE $) + endif() + endif () + else () + find_package (HDF5) # Legacy find + #Legacy find_package does not set HDF5_TOOLS_DIR, so we set it here + set (HDF5_TOOLS_DIR ${HDF5_LIBRARY_DIRS}/../bin) + #Legacy find_package does not set HDF5_BUILD_SHARED_LIBS, so we set it here + if (USE_SHARED_LIBS AND EXISTS "${HDF5_LIBRARY_DIRS}/libhdf5.so") + set (HDF5_BUILD_SHARED_LIBS 1) + else () + set (HDF5_BUILD_SHARED_LIBS 0) + endif () + set (H5PL_HDF5_LINK_LIBS ${H5PL_HDF5_LINK_LIBS} ${HDF5_LIBRARIES}) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${HDF5_LIBRARIES}) + endif () + add_executable (${HDF5_NAMESPACE}h5dump IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5dump PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5dump") + set (H5PL_HDF5_DUMP_EXECUTABLE $) + add_executable (${HDF5_NAMESPACE}h5repack IMPORTED) + set_property (TARGET ${HDF5_NAMESPACE}h5repack PROPERTY IMPORTED_LOCATION "${HDF5_TOOLS_DIR}/h5repack") + set (H5PL_HDF5_REPACK_EXECUTABLE $) + endif () + + set (HDF5_PACKAGE_NAME ${SEARCH_PACKAGE_NAME}) + + if (HDF5_FOUND) + set (H5PL_HDF5_INCLUDE_DIRS ${HDF5_INCLUDE_DIR}) + set (H5PL_HDF5_HAVE_H5PUBCONF_H 1) + set (H5PL_HDF5_HAVE_HDF5 1) + set (H5PL_HDF5_HEADER "h5pubconf.h") + message (STATUS "HDF5-${HDF5_VERSION_STRING} found: INC=${HDF5_INCLUDE_DIR} TOOLS=${HDF5_TOOLS_DIR}") + else () + message (FATAL_ERROR " HDF5 is Required for plugin library") + endif () + else () + # This project is being called from within another and HDF5 is already configured + set (H5PL_HDF5_HAVE_H5PUBCONF_H 1) + set (H5PL_HDF5_HAVE_HDF5 1) + if (${link_hdf}) + #plugin source needs to be linked with HDF5 + set (H5PL_LINK_LIBS ${H5PL_LINK_LIBS} ${H5PL_HDF5_LINK_LIBS}) + endif () + set (USE_SHARED_LIBS ON CACHE BOOL "Use Shared Libraries" FORCE) + endif () + message (STATUS "HDF5 link libs: ${H5PL_LINK_LIBS} Includes: ${H5PL_HDF5_INCLUDE_DIRS}") +endmacro () diff --git a/SZ3/config/cmake/binex/config/cmake/cacheinit.cmake b/SZ3/config/cmake/binex/config/cmake/cacheinit.cmake new file mode 100755 index 00000000..97df919d --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake/cacheinit.cmake @@ -0,0 +1,22 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# This is the CMakeCache file. + +######################## +# EXTERNAL cache entries +######################## + +set (H5PL_BUILD_TESTING ON CACHE BOOL "Build HDF5 Unit Testing" FORCE) + +set (HDF5_PACKAGE_NAME "hdf5" CACHE STRING "Name of HDF5 package" FORCE) + +set (HDF5_NAMESPACE "hdf5::" CACHE STRING "Name space of HDF5 library" FORCE) diff --git a/SZ3/config/cmake/binex/config/cmake/grepTest.cmake b/SZ3/config/cmake/binex/config/cmake/grepTest.cmake new file mode 100644 index 00000000..2c83b3c8 --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake/grepTest.cmake @@ -0,0 +1,199 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# grepTest.cmake executes a command and captures the output in a file. File is then compared +# against a reference file. Exit status of command can also be compared. +cmake_policy(SET CMP0007 NEW) + +# arguments checking +if (NOT TEST_PROGRAM) + message (FATAL_ERROR "Require TEST_PROGRAM to be defined") +endif () +if (NOT TEST_FOLDER) + message (FATAL_ERROR "Require TEST_FOLDER to be defined") +endif () +if (NOT TEST_OUTPUT) + message (FATAL_ERROR "Require TEST_OUTPUT to be defined") +endif () +if (NOT TEST_FILTER) + message (STATUS "Optional TEST_FILTER to be defined") +endif () +if (NOT TEST_REFERENCE) + message (FATAL_ERROR "Require TEST_REFERENCE to be defined") +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}) +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err) +endif () + +message (STATUS "COMMAND: ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS}") + +if (TEST_LIBRARY_DIRECTORY) + if (WIN32) + set (ENV{PATH} "$ENV{PATH};${TEST_LIBRARY_DIRECTORY}") + elseif (APPLE) + set (ENV{DYLD_LIBRARY_PATH} "$ENV{DYLD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + else () + set (ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + endif () + set (ENV{HDF5_PLUGIN_PATH} "${TEST_LIBRARY_DIRECTORY}/plugins:${TEST_LIBRARY_DIRECTORY}/plugin:$ENV{HDF5_PLUGIN_PATH}") +endif () + +if (TEST_ENV_VAR) + set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}") + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (TRACE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}") + endif () +endif () + +# run the test program, capture the stdout/stderr and the result var +execute_process ( + COMMAND ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS} + WORKING_DIRECTORY ${TEST_FOLDER} + RESULT_VARIABLE TEST_RESULT + OUTPUT_FILE ${TEST_OUTPUT} + ERROR_FILE ${TEST_OUTPUT}.err + OUTPUT_VARIABLE TEST_OUT + ERROR_VARIABLE TEST_ERROR +) + +message (STATUS "COMMAND Result: ${TEST_RESULT}") + +message (STATUS "COMMAND Error: ${TEST_ERROR}") + +# remove special output +file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) +string (FIND TEST_STREAM "_pmi_alps" TEST_FIND_RESULT) +if (TEST_FIND_RESULT GREATER -1) + string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_STREAM}) +endif () + +# if the TEST_ERRREF exists grep the error output with the error reference +set (TEST_ERRREF_RESULT 0) +if (TEST_ERRREF) + # if the .err file exists grep the error output with the error reference before comparing stdout + if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_ERR_STREAM) + list (LENGTH TEST_ERR_STREAM test_len) + if (test_len GREATER 0) + # TEST_ERRREF should always be matched + string (REGEX MATCH "${TEST_ERRREF}" TEST_MATCH ${TEST_ERR_STREAM}) + string (COMPARE EQUAL "${TEST_ERRREF}" "${TEST_MATCH}" TEST_ERRREF_RESULT) + if (NOT TEST_ERRREF_RESULT) + message (FATAL_ERROR "Failed: The error output of ${TEST_PROGRAM} did not contain ${TEST_ERRREF}") + endif () + endif () + endif () + + #always compare output file to reference unless this must be skipped + set (TEST_COMPARE_RESULT 0) + if (NOT TEST_SKIP_COMPARE) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}") + file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + if (WIN32) + configure_file(${TEST_FOLDER}/${TEST_REFERENCE} ${TEST_FOLDER}/${TEST_REFERENCE}.tmp NEWLINE_STYLE CRLF) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}.tmp") + file(RENAME ${TEST_FOLDER}/${TEST_REFERENCE}.tmp ${TEST_FOLDER}/${TEST_REFERENCE}) + endif () + #file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + #file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") + endif () + if (NOT TEST_SORT_COMPARE) + # now compare the output with the reference + execute_process ( + COMMAND ${CMAKE_COMMAND} -E compare_files ${CMAKE_IGNORE_EOL} ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE} + RESULT_VARIABLE TEST_COMPARE_RESULT + ) + else () + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2) + list (SORT v1) + list (SORT v2) + if (NOT v1 STREQUAL v2) + set (TEST_COMPARE_RESULT 1) + endif () + endif () + + if (TEST_COMPARE_RESULT) + set (TEST_COMPARE_RESULT 0) + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act) + list (LENGTH test_act len_act) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref) + list (LENGTH test_ref len_ref) + if (len_act GREATER 0 AND len_ref GREATER 0) + math (EXPR _FP_LEN "${len_ref} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + list (GET test_act ${line} str_act) + list (GET test_ref ${line} str_ref) + if (NOT str_act STREQUAL str_ref) + if (str_act) + set (TEST_COMPARE_RESULT 1) + message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") + endif () + endif () + endforeach () + else () + if (len_act EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty") + endif () + if (len_ref EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty") + endif () + endif () + if (NOT len_act EQUAL len_ref) + set (TEST_COMPARE_RESULT 1) + endif () + endif () + endif () + + message (STATUS "COMPARE Result: ${TEST_COMPARE_RESULT}") + + # again, if return value is !=0 scream and shout + if (TEST_COMPARE_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}") + endif () + endif () + endif () +else () + # else grep the output with the reference + set (TEST_GREP_RESULT 0) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + + # TEST_REFERENCE should always be matched + string (REGEX MATCH "${TEST_REFERENCE}" TEST_MATCH ${TEST_STREAM}) + string (COMPARE EQUAL "${TEST_REFERENCE}" "${TEST_MATCH}" TEST_GREP_RESULT) + if (NOT TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}") + endif () +endif () + + +if (TEST_FILTER) + string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) + if (TEST_EXPECT) + # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match + string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) + if (TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") + endif () + endif () +endif () + +# everything went fine... +message (STATUS "Passed: The output of ${TEST_PROGRAM} matched") + diff --git a/SZ3/config/cmake/binex/config/cmake/runTest.cmake b/SZ3/config/cmake/binex/config/cmake/runTest.cmake new file mode 100644 index 00000000..edf46a86 --- /dev/null +++ b/SZ3/config/cmake/binex/config/cmake/runTest.cmake @@ -0,0 +1,387 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# runTest.cmake executes a command and captures the output in a file. File is then compared +# against a reference file. Exit status of command can also be compared. +cmake_policy(SET CMP0007 NEW) + +# arguments checking +if (NOT TEST_PROGRAM) + message (FATAL_ERROR "Require TEST_PROGRAM to be defined") +endif () +if (NOT TEST_FOLDER) + message (FATAL_ERROR "Require TEST_FOLDER to be defined") +endif () +if (NOT TEST_OUTPUT) + message (FATAL_ERROR "Require TEST_OUTPUT to be defined") +endif () +if (NOT DEFINED TEST_EXPECT) + message (STATUS "Require TEST_EXPECT to be defined") +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}) +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err) +endif () + +message (STATUS "COMMAND: ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS}") + +if (TEST_LIBRARY_DIRECTORY) + if (WIN32) + set (ENV{PATH} "$ENV{PATH};${TEST_LIBRARY_DIRECTORY}") + elseif (APPLE) + set (ENV{DYLD_LIBRARY_PATH} "$ENV{DYLD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + else () + set (ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + endif () + set (ENV{HDF5_PLUGIN_PATH} "${TEST_LIBRARY_DIRECTORY}/plugins:${TEST_LIBRARY_DIRECTORY}/plugin:$ENV{HDF5_PLUGIN_PATH}") +endif () + +if (TEST_ENV_VAR) + set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}") + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (TRACE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}") + endif () +endif () + +if (NOT TEST_INPUT) + # run the test program, capture the stdout/stderr and the result var + execute_process ( + COMMAND ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS} + WORKING_DIRECTORY ${TEST_FOLDER} + RESULT_VARIABLE TEST_RESULT + OUTPUT_FILE ${TEST_OUTPUT} + ERROR_FILE ${TEST_OUTPUT}.err + OUTPUT_VARIABLE TEST_OUT + ) +else () + # run the test program with stdin, capture the stdout/stderr and the result var + execute_process ( + COMMAND ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS} + WORKING_DIRECTORY ${TEST_FOLDER} + RESULT_VARIABLE TEST_RESULT + INPUT_FILE ${TEST_INPUT} + OUTPUT_FILE ${TEST_OUTPUT} + ERROR_FILE ${TEST_OUTPUT}.err + OUTPUT_VARIABLE TEST_OUT + ) +endif () + +if (TEST_REGEX) + # TEST_REGEX should always be matched + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + string (REGEX MATCH "${TEST_REGEX}" REGEX_MATCH ${TEST_STREAM}) + string (COMPARE EQUAL "${REGEX_MATCH}" "${TEST_MATCH}" REGEX_RESULT) + if (NOT REGEX_RESULT) + message (STATUS "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_MATCH}") + endif () +endif () + +message (STATUS "COMMAND Result: ${TEST_RESULT}") + +# if the .err file exists and ERRROR_APPEND is enabled +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_ERROR) + list (LENGTH TEST_ERROR test_len) + if (test_len GREATER 0) + if (TEST_MASK_FILE) + STRING(REGEX REPLACE "CurrentDir is [^\n]+\n" "CurrentDir is (dir name)\n" TEST_ERROR "${TEST_ERROR}") + endif () + # remove special output + string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_ERROR "${TEST_ERROR}") + + if (NOT ERROR_APPEND) + # write back to original .err file + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_ERROR}") + else () + # append error output to the stdout output file + file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_ERROR}") + endif () + endif () +endif () + +# append the test result status with a predefined text +if (TEST_APPEND) + file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_APPEND} ${TEST_RESULT}\n") +endif () + +# if the return value is !=${TEST_EXPECT} bail out +if (NOT TEST_RESULT EQUAL TEST_EXPECT) + if (NOT TEST_NOERRDISPLAY) + if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + message (STATUS "Output :\n${TEST_STREAM}") + endif () + endif () + message (FATAL_ERROR "Failed: Test program ${TEST_PROGRAM} exited != ${TEST_EXPECT}.\n${TEST_ERROR}") +endif () + +message (STATUS "COMMAND Error: ${TEST_ERROR}") + +# remove special output +file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) +string (FIND TEST_STREAM "_pmi_alps" TEST_FIND_RESULT) +if (TEST_FIND_RESULT GREATER -1) + string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_STREAM}) +endif () + +# remove special error output +if (NOT TEST_ERRREF) + # the error stack has been appended to the output file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) +else () + # the error stack remains in the .err file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM) +endif () +string (FIND TEST_STREAM "no version information available" TEST_FIND_RESULT) +if (TEST_FIND_RESULT GREATER -1) + string (REGEX REPLACE "^.*no version information available[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}") + # write back the changes to the original files + if (NOT TEST_ERRREF) + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") + else () + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}") + endif () +endif () + +# if the output file needs COMPRESSION text modified +if (TEST_MASK_MOD) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + string (REGEX REPLACE " SIZE [0-9]* .([0-9]*)[.][0-9]*:1 COMPRESSION." " SIZE XXX (\\1.XXXX:1 COMPRESSION)" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") +endif () + +# if the output file or the .err file needs to mask out error stack info +if (TEST_MASK_ERROR) + if (NOT TEST_ERRREF) + # the error stack has been appended to the output file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + else () + # the error stack remains in the .err file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM) + endif () + string (REGEX REPLACE "thread [0-9]*:" "thread (IDs):" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE ": ([^\n]*)[.]c " ": (file name) " TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE " line [0-9]*" " line (number)" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "v[1-9]*[.][0-9]*[.]" "version (number)." TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "[1-9]*[.][0-9]*[.][0-9]*[^)]*" "version (number)" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "H5Eget_auto[1-2]*" "H5Eget_auto(1 or 2)" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "H5Eset_auto[1-2]*" "H5Eset_auto(1 or 2)" TEST_STREAM "${TEST_STREAM}") + # write back the changes to the original files + if (NOT TEST_ERRREF) + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") + else () + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}") + endif () +endif () + +# remove text from the output file +if (TEST_FILTER) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + string (REGEX REPLACE "${TEST_FILTER}" "${TEST_FILTER_REPLACE}" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") +endif () + +if (TEST_REF_FILTER) + #message (STATUS "TEST_REF_FILTER: ${TEST_APPEND}${TEST_REF_FILTER}") + file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + string (REGEX REPLACE "${TEST_REF_APPEND}" "${TEST_REF_FILTER}" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") +endif () + +# compare output files to references unless this must be skipped +set (TEST_COMPARE_RESULT 0) +if (NOT TEST_SKIP_COMPARE) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}") + file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + if (WIN32) + configure_file (${TEST_FOLDER}/${TEST_REFERENCE} ${TEST_FOLDER}/${TEST_REFERENCE}.tmp NEWLINE_STYLE CRLF) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}.tmp") + file(RENAME ${TEST_FOLDER}/${TEST_REFERENCE}.tmp ${TEST_FOLDER}/${TEST_REFERENCE}) + endif () + #file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + #file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") + endif () + + if (NOT TEST_SORT_COMPARE) + # now compare the output with the reference + execute_process ( + COMMAND ${CMAKE_COMMAND} -E compare_files ${CMAKE_IGNORE_EOL} ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE} + RESULT_VARIABLE TEST_COMPARE_RESULT + ) + else () + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2) + list (SORT v1) + list (SORT v2) + if (NOT v1 STREQUAL v2) + set (TEST_COMPARE_RESULT 1) + endif () + endif () + + if (TEST_COMPARE_RESULT) + set (TEST_COMPARE_RESULT 0) + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act) + list (LENGTH test_act len_act) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref) + list (LENGTH test_ref len_ref) + if (NOT len_act EQUAL len_ref) + set (TEST_COMPARE_RESULT 1) + endif () + if (len_act GREATER 0 AND len_ref GREATER 0) + if (TEST_SORT_COMPARE) + list (SORT test_act) + list (SORT test_ref) + endif () + math (EXPR _FP_LEN "${len_ref} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + if (line GREATER_EQUAL len_act) + message (STATUS "COMPARE FAILED: ran out of lines in ${TEST_FOLDER}/${TEST_OUTPUT}") + set (TEST_COMPARE_RESULT 1) + break () + elseif (line GREATER_EQUAL len_ref) + message (STATUS "COMPARE FAILED: ran out of lines in ${TEST_FOLDER}/${TEST_REFERENCE}") + set (TEST_COMPARE_RESULT 1) + break () + else () + list (GET test_act ${line} str_act) + list (GET test_ref ${line} str_ref) + if (NOT str_act STREQUAL str_ref) + if (str_act) + set (TEST_COMPARE_RESULT 1) + message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") + endif () + endif () + endif () + endforeach () + else () + if (len_act EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty") + endif () + if (len_ref EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty") + endif () + endif () + endif () + endif () + + message (STATUS "COMPARE Result: ${TEST_COMPARE_RESULT}") + + # again, if return value is !=0 scream and shout + if (TEST_COMPARE_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}") + endif () + endif () + + # now compare the .err file with the error reference, if supplied + set (TEST_ERRREF_RESULT 0) + if (TEST_ERRREF) + file (READ ${TEST_FOLDER}/${TEST_ERRREF} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + if (WIN32) + configure_file (${TEST_FOLDER}/${TEST_ERRREF} ${TEST_FOLDER}/${TEST_ERRREF}.tmp NEWLINE_STYLE CRLF) + if (EXISTS "${TEST_FOLDER}/${TEST_ERRREF}.tmp") + file(RENAME ${TEST_FOLDER}/${TEST_ERRREF}.tmp ${TEST_FOLDER}/${TEST_ERRREF}) + endif () + #file (READ ${TEST_FOLDER}/${TEST_ERRREF} TEST_STREAM) + #file (WRITE ${TEST_FOLDER}/${TEST_ERRREF} "${TEST_STREAM}") + endif () + + # now compare the error output with the error reference + execute_process ( + COMMAND ${CMAKE_COMMAND} -E compare_files ${CMAKE_IGNORE_EOL} ${TEST_FOLDER}/${TEST_OUTPUT}.err ${TEST_FOLDER}/${TEST_ERRREF} + RESULT_VARIABLE TEST_ERRREF_RESULT + ) + if (TEST_ERRREF_RESULT) + set (TEST_ERRREF_RESULT 0) + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT}.err test_act) + list (LENGTH test_act len_act) + file (STRINGS ${TEST_FOLDER}/${TEST_ERRREF} test_ref) + list (LENGTH test_ref len_ref) + math (EXPR _FP_LEN "${len_ref} - 1") + if (len_act GREATER 0 AND len_ref GREATER 0) + math (EXPR _FP_LEN "${len_ref} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + list (GET test_act ${line} str_act) + list (GET test_ref ${line} str_ref) + if (NOT str_act STREQUAL str_ref) + if (str_act) + set (TEST_ERRREF_RESULT 1) + message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") + endif () + endif () + endforeach () + else () + if (len_act EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT}.err is empty") + endif () + if (len_ref EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_ERRREF} is empty") + endif () + endif () + if (NOT len_act EQUAL len_ref) + set (TEST_ERRREF_RESULT 1) + endif () + endif () + endif () + + message (STATUS "COMPARE Result: ${TEST_ERRREF_RESULT}") + + # again, if return value is !=0 scream and shout + if (TEST_ERRREF_RESULT) + message (FATAL_ERROR "Failed: The error output of ${TEST_OUTPUT}.err did not match ${TEST_ERRREF}") + endif () + endif () +endif () + +set (TEST_GREP_RESULT 0) +if (TEST_GREP_COMPARE) + # now grep the output with the reference + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + # TEST_REFERENCE should always be matched + string (REGEX MATCH "${TEST_REFERENCE}" TEST_MATCH ${TEST_STREAM}) + string (COMPARE EQUAL "${TEST_REFERENCE}" "${TEST_MATCH}" TEST_GREP_RESULT) + if (NOT TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}") + endif () + + string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) + if (TEST_EXPECT) + # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match + string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) + if (TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") + endif () + endif () + endif () +endif () + +# dump the output unless nodisplay option is set +if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + execute_process ( + COMMAND ${CMAKE_COMMAND} -E echo ${TEST_STREAM} + RESULT_VARIABLE TEST_RESULT + ) +endif () + +# everything went fine... +message (STATUS "${TEST_PROGRAM} Passed") + diff --git a/SZ3/config/cmake/binex/config/toolchain/aarch64.cmake b/SZ3/config/cmake/binex/config/toolchain/aarch64.cmake new file mode 100644 index 00000000..aa84a742 --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/aarch64.cmake @@ -0,0 +1,17 @@ +set(TOOLCHAIN_PREFIX aarch64-linux-gnu) +set(ANDROID_NDK /opt/android-ndk-linux) +set (CMAKE_SYSTEM_NAME Android) +set (CMAKE_ANDROID_ARCH_ABI x86_64) +#set (CMAKE_ANDROID_STANDALONE_TOOLCHAIN ${ANDROID_NDK}/build/cmake/andriod.toolchain.cmake) +set (CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set (CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +#set (CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) +set (CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) +set (CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) +set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set (CMAKE_CROSSCOMPILING_EMULATOR qemu-aarch64) + +include_directories(/usr/${TOOLCHAIN_PREFIX}/include) + diff --git a/SZ3/config/cmake/binex/config/toolchain/build32.cmake b/SZ3/config/cmake/binex/config/toolchain/build32.cmake new file mode 100644 index 00000000..43ac5edb --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/build32.cmake @@ -0,0 +1,90 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +if (WIN32) + set (CMAKE_SYSTEM_NAME Windows) + set (CMAKE_GENERATOR_PLATFORM "x86") +elseif(APPLE) + set (CMAKE_OSX_ARCHITECTURES "i386") +elseif(MINGW) + set (CMAKE_SYSTEM_NAME Windows) + set (TOOLCHAIN_PREFIX i686-w64-mingw32) + set (CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) + set (CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) + set (CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + set (CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags") + + set (LIB32 /usr/lib) # Fedora + + if (EXISTS "/usr/lib32") + set (LIB32 /usr/lib32) # Arch, Solus + endif () + + set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE) + set (CMAKE_LIBRARY_PATH ${LIB32} CACHE STRING "library search path" FORCE) + + # this is probably unlikely to be needed, but just in case + set (CMAKE_EXE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "executable linker flags" FORCE) + set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE) + set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags" FORCE) + + # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have + # to include the regular system .pc files as well (at the end), because some + # are not always present in the 32 bit directory + if (EXISTS "${LIB32}/pkgconfig") + set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig) + endif () + + set (CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + set (CMAKE_CROSSCOMPILING_EMULATOR wine) + + include_directories(/usr/${TOOLCHAIN_PREFIX}/include) + set (CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS On CACHE BOOL "Export windows symbols") +else () + set (CMAKE_SYSTEM_NAME Linux) + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags") + + set (LIB32 /usr/lib) # Fedora + + if (EXISTS "/usr/lib32") + set (LIB32 /usr/lib32) # Arch, Solus + endif () + + set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE) + set (CMAKE_LIBRARY_PATH ${LIB32} CACHE STRING "library search path" FORCE) + + # this is probably unlikely to be needed, but just in case + set (CMAKE_EXE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "executable linker flags" FORCE) + set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE) + set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags" FORCE) + + # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have + # to include the regular system .pc files as well (at the end), because some + # are not always present in the 32 bit directory + if (EXISTS "${LIB32}/pkgconfig") + set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig) + endif () +# where is the target environment + set (CMAKE_FIND_ROOT_PATH ${LIB32}) +# search for programs in the build host directories + set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +# for libraries and headers in the target directories + set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endif () diff --git a/SZ3/config/cmake/binex/config/toolchain/clang.cmake b/SZ3/config/cmake/binex/config/toolchain/clang.cmake new file mode 100644 index 00000000..1242e134 --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/clang.cmake @@ -0,0 +1,27 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Uncomment the following to use cross-compiling +#set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "clang") + +if(WIN32) + set (CMAKE_C_COMPILER clang-cl) + set (CMAKE_CXX_COMPILER clang-cl) +else() + set (CMAKE_C_COMPILER clang) + set (CMAKE_CXX_COMPILER clang++) +endif() +set (CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/cmake/binex/config/toolchain/crayle.cmake b/SZ3/config/cmake/binex/config/toolchain/crayle.cmake new file mode 100644 index 00000000..52248903 --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/crayle.cmake @@ -0,0 +1,21 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# The following line will use cross-compiling +set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "CrayLinuxEnvironment") + +set (CMAKE_C_COMPILER cc) +set (CMAKE_Fortran_COMPILER ftn) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/cmake/binex/config/toolchain/gcc.cmake b/SZ3/config/cmake/binex/config/toolchain/gcc.cmake new file mode 100644 index 00000000..7f3c7586 --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/gcc.cmake @@ -0,0 +1,22 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Uncomment the following line and the correct system name to use cross-compiling +#set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "GCC") + +set (CMAKE_C_COMPILER cc) +set (CMAKE_CXX_COMPILER c++) +set (CMAKE_Fortran_COMPILER gfortran) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/cmake/binex/config/toolchain/icc.cmake b/SZ3/config/cmake/binex/config/toolchain/icc.cmake new file mode 100644 index 00000000..97f6a649 --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/icc.cmake @@ -0,0 +1,11 @@ +# Uncomment the following to use cross-compiling +#set(CMAKE_SYSTEM_NAME Linux) + +set(CMAKE_COMPILER_VENDOR "intel") + +set(CMAKE_C_COMPILER icc) +set(CMAKE_CXX_COMPILER icpc) +set(CMAKE_Fortran_COMPILER ifort) + +# the following is used if cross-compiling +set(CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/cmake/binex/config/toolchain/mingw64.cmake b/SZ3/config/cmake/binex/config/toolchain/mingw64.cmake new file mode 100644 index 00000000..07517b44 --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/mingw64.cmake @@ -0,0 +1,25 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +set (TOOLCHAIN_PREFIX x86_64-w64-mingw32) +set (CMAKE_SYSTEM_NAME Windows) +set (CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set (CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set (CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) +set (CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) +set (CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) +set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set (CMAKE_CROSSCOMPILING_EMULATOR wine64) + +include_directories(/usr/${TOOLCHAIN_PREFIX}/include) +set (CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS On CACHE BOOL "Export windows symbols") diff --git a/SZ3/config/cmake/binex/config/toolchain/pgi.cmake b/SZ3/config/cmake/binex/config/toolchain/pgi.cmake new file mode 100644 index 00000000..de67f59a --- /dev/null +++ b/SZ3/config/cmake/binex/config/toolchain/pgi.cmake @@ -0,0 +1,22 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Uncomment the following to use cross-compiling +#set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "PGI") + +set (CMAKE_C_COMPILER pgcc) +set (CMAKE_CXX_COMPILER pgc++) +set (CMAKE_Fortran_COMPILER pgf90) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/cmake/binex/example/CMakeLists.txt b/SZ3/config/cmake/binex/example/CMakeLists.txt new file mode 100644 index 00000000..6a985885 --- /dev/null +++ b/SZ3/config/cmake/binex/example/CMakeLists.txt @@ -0,0 +1,173 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +cmake_minimum_required (VERSION 3.18) +project (HDFPL_EX C) + +set (dyn_examples + h5ex_d_sz3 +) + +if (USE_SHARED_LIBS) + add_definitions (-DH5_BUILT_AS_DYNAMIC_LIB) +else () + add_definitions (-DH5_BUILT_AS_STATIC_LIB) +endif () + +#run-time loadable sz3 library examples +foreach (example ${dyn_examples}) + add_executable (${example} ${PROJECT_SOURCE_DIR}/${example}.c) + target_include_directories(${example} PRIVATE ${H5PL_HDF5_INCLUDE_DIRS}) + TARGET_C_PROPERTIES (${example} ${LIB_TYPE}) + target_link_libraries (${example} PRIVATE ${H5PL_HDF5_LINK_LIBS}) + if (NOT WIN32) + target_link_libraries (${example} PRIVATE ${CMAKE_DL_LIBS}) + endif () +endforeach () + +if (H5PL_BUILD_TESTING) + macro (ADD_H5_TEST testname) + add_test ( + NAME ${testname}-clearall + COMMAND ${CMAKE_COMMAND} + -E remove + ${testname}.out + ${testname}.out.err + ${testname}.ddl.out + ${testname}.ddl.out.err + ${testname}.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (${testname}-clearall PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "${testname}-clearall") + if (WIN32) + set (TESTLIBDIR "${HDF5_TOOLS_DIR}") + else () + set (TESTLIBDIR "${HDF5_LIBRARY_PATH}") + endif () + add_test ( + NAME ${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_REFERENCE=${testname}.tst" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (${testname} PROPERTIES DEPENDS ${testname}-clearall) + set (last_test "${testname}") + if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) + add_test ( + NAME H5DUMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=--enable-error-stack;-p;${testname}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${testname}.ddl.out" + -D "TEST_EXPECT=0" + -D "TEST_REFERENCE=${testname}.ddl" + -D "TEST_FILTER:STRING=PARAMS {[ -0-9]*}" + -D "TEST_FILTER_REPLACE:STRING=PARAMS { XXXX }" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5DUMP-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS ${testname}) + set (last_test "H5DUMP-${testname}") + endif () + endmacro () + + macro (ADD_H5_UD_TEST testname resultcode resultfile) + # Remove any output file left over from previous test run + add_test ( + NAME H5SZ3_UD-${testname}-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + out-${testname}.${resultfile} + ${testname}.${resultfile}.out + ${testname}.${resultfile}.out.err + ${resultfile}-${testname}.out + ${resultfile}-${testname}.out.err + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5SZ3_UD-${testname}-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5SZ3_UD-${testname}-clearall-objects") + if (WIN32) + set (${TESTLIBDIR} "${HDF5_TOOLS_DIR}") + else () + set (${TESTLIBDIR} "${HDF5_LIBRARY_PATH}") + endif () + if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) + add_test ( + NAME H5SZ3_UD-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_REPACK_EXECUTABLE}" + -D "TEST_ARGS:STRING=${ARGN};${PROJECT_BINARY_DIR}/${resultfile};${PROJECT_BINARY_DIR}/out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_FILTER:STRING=O?...ing file[^\n]+\n" + -D "TEST_OUTPUT=${testname}.${resultfile}.out" + -D "TEST_REFERENCE=${testname}.${resultfile}.tst" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5SZ3_UD-${testname} PROPERTIES DEPENDS H5SZ3_UD-${testname}-clearall-objects) + add_test ( + NAME H5SZ3_UD-h5dump-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=-pH;out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}-${testname}.ddl" + -D "TEST_FILTER:STRING=PARAMS {[ -0-9]*}" + -D "TEST_FILTER_REPLACE:STRING=PARAMS { XXXX }" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -P "${H5PL_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5SZ3_UD-h5dump-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5SZ3_UD-${testname}) + set (last_test "H5SZ3_UD-h5dump-${testname}") + endif () + endmacro () + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (LIST_HDF5_TEST_FILES + h5repack_layout.h5 + ) + set (LIST_OTHER_TEST_FILES + h5ex_d_sz3.ddl + h5ex_d_sz3.tst + h5repack_layout.h5-ud_convert.ddl + ud_convert.h5repack_layout.h5.tst + ) + + foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "example_files") + endforeach () + add_custom_target(example_files ALL COMMENT "Copying files needed by example tests" DEPENDS ${example_files_list}) + + ADD_H5_TEST (h5ex_d_sz3) + + #UD SZ3 with the default settings (ALGO_INTERP_LORENZO, absolute error bound 1e-3) + ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32024,0) + +endif () diff --git a/SZ3/config/cmake/binex/example/h5ex_d_sz3.c b/SZ3/config/cmake/binex/example/h5ex_d_sz3.c new file mode 100644 index 00000000..8a40d3ee --- /dev/null +++ b/SZ3/config/cmake/binex/example/h5ex_d_sz3.c @@ -0,0 +1,253 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 SZ3 filter plugin source. The full * + * copyright notice, including terms governing use, modification, and * + * terms governing use, modification, and redistribution, is contained in * + * the file COPYING, which can be found at the root of the SZ3 source code * + * distribution tree. If you do not have access to this file, you may * + * request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/************************************************************ + + This example shows how to write data and read it from a dataset + using SZ3 compression. + SZ3 filter is not available in HDF5. + The example uses a new feature available in HDF5 version 1.8.11 + to discover, load and register filters at run time. + + No cd_values are passed, so the filter's set_local callback + stores SZ3's default settings: the ALGO_INTERP_LORENZO + algorithm with an absolute error bound of 1e-3. SZ3 is lossy, + so the data read back is checked against that bound. + + ************************************************************/ + +#include "hdf5.h" +#include +#include + +#define FILENAME "h5ex_d_sz3.h5" +#define DATASET "DS1" +#define DIM0 32 +#define DIM1 64 +#define CHUNK0 4 +#define CHUNK1 8 +#define H5Z_FILTER_SZ3 32024 +#define SZ3_ABS_BOUND 1e-3 /* SZ3's default absolute error bound */ + +int +main(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + herr_t status; + htri_t avail; + H5Z_filter_t filter_id = 0; + char filter_name[256]; + hsize_t dims[2] = {DIM0, DIM1}, chunk[2] = {CHUNK0, CHUNK1}; + size_t nelmts = 10; /* number of elements in values_out */ + unsigned int flags; + unsigned filter_config; + unsigned int values_out[10] = {99, 99, 99, 99, 99, 99, 99, 99, 99, 99}; + float wdata[DIM0][DIM1]; /* Write buffer */ + float rdata[DIM0][DIM1]; /* Read buffer */ + float max; + double err, max_err; + hsize_t i, j; + int ret_value = 1; + + /* + * Initialize data. + */ + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) + wdata[i][j] = (float)(i * j) - (float)(j); + + /* + * Create a new file using the default properties. + */ + file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file_id < 0) + goto done; + + /* + * Create dataspace. Setting maximum size to NULL sets the maximum + * size to be the current size. + */ + space_id = H5Screate_simple(2, dims, NULL); + if (space_id < 0) + goto done; + + /* + * Create the dataset creation property list, add the SZ3 + * compression filter and set the chunk size. + */ + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) + goto done; + + status = H5Pset_filter(dcpl_id, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, 0, NULL); + if (status < 0) + goto done; + + /* + * Check that filter is registered with the library now. + * If it is registered, retrieve filter's configuration. + */ + avail = H5Zfilter_avail(H5Z_FILTER_SZ3); + if (avail) { + status = H5Zget_filter_info(H5Z_FILTER_SZ3, &filter_config); + if ((filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED) && + (filter_config & H5Z_FILTER_CONFIG_DECODE_ENABLED)) + printf("sz3 filter is available for encoding and decoding.\n"); + } + else { + printf("H5Zfilter_avail - not found.\n"); + goto done; + } + status = H5Pset_chunk(dcpl_id, 2, chunk); + if (status < 0) + printf("failed to set chunk.\n"); + + /* + * Create the dataset. + */ + printf("....Create dataset ................\n"); + dset_id = H5Dcreate(file_id, DATASET, H5T_IEEE_F32LE, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dset_id < 0) { + printf("failed to create dataset.\n"); + goto done; + } + + /* + * Write the data to the dataset. + */ + printf("....Writing sz3 compressed data ................\n"); + status = H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]); + if (status < 0) + printf("failed to write data.\n"); + + /* + * Close and release resources. + */ + H5Dclose(dset_id); + dset_id = -1; + H5Pclose(dcpl_id); + dcpl_id = -1; + H5Sclose(space_id); + space_id = -1; + H5Fclose(file_id); + file_id = -1; + status = H5close(); + if (status < 0) { + printf("\nFAILED to close library\n"); + goto done; + } + + printf("....Close the file and reopen for reading ........\n"); + /* + * Now we begin the read section of this example. + */ + + /* + * Open file and dataset using the default properties. + */ + file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) + goto done; + + dset_id = H5Dopen(file_id, DATASET, H5P_DEFAULT); + if (dset_id < 0) + goto done; + + /* + * Retrieve dataset creation property list. + */ + dcpl_id = H5Dget_create_plist(dset_id); + if (dcpl_id < 0) + goto done; + + /* + * Retrieve and print the filter id, number of parameters and filter's name for SZ3. + * The first parameter is the SZ3 data format version, (major << 24) | (minor << 16) | (patch << 8); + * the others hold the SZ3 configuration the filter stored for the dataset. + */ + filter_id = H5Pget_filter2(dcpl_id, (unsigned)0, &flags, &nelmts, values_out, sizeof(filter_name), + filter_name, NULL); + printf("Filter info is available from the dataset creation property\n"); + printf(" Filter identifier is "); + switch (filter_id) { + case H5Z_FILTER_SZ3: + printf("%d\n", filter_id); + printf(" Number of parameters is %d with the value %u\n", (int)nelmts, values_out[0]); + printf(" To find more about the filter check %s\n", filter_name); + break; + default: + printf("Not expected filter\n"); + break; + } + + /* + * Read the data using the default properties. + */ + printf("....Reading sz3 compressed data ................\n"); + status = H5Dread(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata[0]); + if (status < 0) { + printf("failed to read data.\n"); + goto done; + } + + /* + * Find the maximum value in the dataset and the largest error, to verify + * that it was read correctly and within the error bound. + */ + max = rdata[0][0]; + max_err = 0.0; + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) { + if (max < rdata[i][j]) + max = rdata[i][j]; + err = (double)rdata[i][j] - (double)wdata[i][j]; + if (err < 0) + err = -err; + if (err > max_err) + max_err = err; + } + /* + * Print the maximum value, rounded to the error bound. + */ + printf("Maximum value in %s is %6.2f\n", DATASET, max); + if (max_err > SZ3_ABS_BOUND) { + printf("Maximum absolute error %g exceeds the error bound %g\n", max_err, SZ3_ABS_BOUND); + goto done; + } + printf("Maximum absolute error is within the error bound %g\n", SZ3_ABS_BOUND); + /* + * Check that filter is registered with the library now. + */ + avail = H5Zfilter_avail(H5Z_FILTER_SZ3); + if (avail) + printf("sz3 filter is available now since H5Dread triggered loading of the filter.\n"); + + ret_value = 0; + +done: + /* + * Close and release resources. + */ + if (dcpl_id >= 0) + H5Pclose(dcpl_id); + if (dset_id >= 0) + H5Dclose(dset_id); + if (space_id >= 0) + H5Sclose(space_id); + if (file_id >= 0) + H5Fclose(file_id); + + return ret_value; +} diff --git a/SZ3/config/cmake/binex/example/test/test.sh b/SZ3/config/cmake/binex/example/test/test.sh new file mode 100644 index 00000000..aca64a98 --- /dev/null +++ b/SZ3/config/cmake/binex/example/test/test.sh @@ -0,0 +1,258 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + +# This file is for use of h5cc created with the autotools process +# HDF5_HOME is expected to be set + +srcdir=.. +builddir=. +verbose=yes +nerrors=0 + +# HDF5 compile commands, assuming they are in your $PATH. +H5CC=$HDF5_HOME/bin/h5cc +LD_LIBRARY_PATH=$HDF5_HOME/lib +export LD_LIBRARY_PATH + +if ! test -f $H5CC; then + echo "Set paths for H5CC and LD_LIBRARY_PATH in test.sh" + echo "Set environment variable HDF5_HOME to the hdf5 install dir" + echo "h5cc was not found at $H5CC" + exit $EXIT_FAILURE +fi + +case $H5CC in +*/*) H5DUMP=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5dump/'`; + test -x $H5DUMP || H5DUMP=h5dump; + H5REPACK=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5repack/'`; + test -x $H5REPACK || H5REPACK=h5repack;; +*) H5DUMP=h5dump; + H5REPACK=h5repack;; +esac + +# Shell commands used in Makefiles +RM="rm -rf" +DIFF="diff -c" +CMP="cmp -s" +GREP='grep' +CP="cp -p" # Use -p to preserve mode,ownership,timestamps +DIRNAME='dirname' +LS='ls' +AWK='awk' + +# setup plugin path +ENVCMD="env HDF5_PLUGIN_PATH=$LD_LIBRARY_PATH/plugin" + +TESTDIR=$builddir + + +$H5CC -shlib $srcdir/h5ex_d_sz3.c -o h5ex_d_sz3 + +SRC_TESTFILES="$srcdir/testfiles" +LIST_TEST_FILES=" +$SRC_TESTFILES/h5repack_layout.h5 +$SRC_TESTFILES/h5ex_d_sz3.ddl +$SRC_TESTFILES/h5ex_d_sz3.tst +$SRC_TESTFILES/h5repack_layout.h5-ud_convert.ddl +$SRC_TESTFILES/ud_convert.h5repack_layout.h5.tst +" + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES="$LIST_TEST_FILES" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +# Print a $* message left justified in a field of 70 characters +# +MESSAGE() { + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + MESSAGE "Verifying $*" +} + +# This is different from $srcdir/../../bin/output_filter.sh +STDOUT_FILTER() { + result_file=$1 + tmp_file=/tmp/h5test_tmp_$$ + # Filter name of files. + cp $result_file $tmp_file + sed -e '/^Opening file/d' -e '/^Making file/d' \ + < $tmp_file > $result_file + # cleanup + rm -f $tmp_file +} + +# Compare the two text files +# PASS if same +# FAIL if different, and show the diff +# +# Assumed arguments: +# $1 is text file1 (expected output) +# $2 is text file2 (actual output) +CMP_OUTPUT() +{ + expect=$1 + actual=$2 + + VERIFY $@ + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " Expected Missing" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected output differs from actual output" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# ADD_H5_TEST +DUMPTEST() { + expect="$1" + actual="`basename $1 .ddl`.out" + actual_err="`basename $1 .ddl`.err" + shift + + # Run test. + TESTING $DUMPER $@ + ( + $ENVCMD $H5DUMP --enable-error-stack -p "$@" + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual + + # Clean up output file +# rm -f $actual $actual_err + +} + +REPACKTEST() +{ + infile=$1 + outfile=$2 + expect="$2.tst" + actual="$2.out1" + actual_err="$2.err1" + shift + shift + + # Run test. + TESTING H5REPACK $@ + ( + $ENVCMD $H5REPACK "$@" "out-$infile" "out-$outfile" + ) >$actual 2>$actual_err + RET=$? + STDOUT_FILTER $actual + cat $actual_err >> $actual + + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + + CMP_OUTPUT $expect $actual + fi +# rm -f $actual $actual_err +} + + +EXETEST() { + fname=$1 + expect="$2" + actual="`basename $2 .tst`.out1" + actual_err="`basename $2 .tst`.err1" + + # Run test. + TESTING $fname + ( + $ENVCMD `pwd`/$fname + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual +# rm -f $actual $actual_err +} + +################################ +######### THE TESTS ######### +################################ + +echo $ENVCMD + +COPY_TESTFILES_TO_TESTDIR + +EXETEST h5ex_d_sz3 h5ex_d_sz3.tst + +DUMPTEST h5ex_d_sz3.ddl h5ex_d_sz3.h5 + +REPACKTEST h5repack_layout.h5 out-ud_convert.h5repack_layout.h5 -v -f UD=32024,0 + +DUMPTEST h5repack_layout.h5-ud_convert.ddl -pH out-ud_convert.h5repack_layout.h5 + +#rm -f ud_convert.h5repack_layout.h5 + +echo "$nerrors tests failed in example" +exit $nerrors diff --git a/SZ3/config/cmake/binex/example/test/testCM.sh b/SZ3/config/cmake/binex/example/test/testCM.sh new file mode 100755 index 00000000..1e3a2551 --- /dev/null +++ b/SZ3/config/cmake/binex/example/test/testCM.sh @@ -0,0 +1,258 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + +# This file is for use of h5cc created with the CMake process +# HDF5_HOME is expected to be set + +srcdir=.. +builddir=. +verbose=yes +nerrors=0 + +# HDF5 compile commands, assuming they are in your $PATH. +H5CC=$HDF5_HOME/bin/h5cc +LD_LIBRARY_PATH=$HDF5_HOME/lib +export LD_LIBRARY_PATH + +if ! test -f $H5CC; then + echo "Set paths for H5CC and LD_LIBRARY_PATH in test.sh" + echo "Set environment variable HDF5_HOME to the hdf5 install dir" + echo "h5cc was not found at $H5CC" + exit $EXIT_FAILURE +fi + +case $H5CC in +*/*) H5DUMP=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5dump-shared/'`; + test -x $H5DUMP || H5DUMP=h5dump-shared; + H5REPACK=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5repack-shared/'`; + test -x $H5REPACK || H5REPACK=h5repack-shared;; +*) H5DUMP=h5dump-shared; + H5REPACK=h5repack-shared;; +esac + +# Shell commands used in Makefiles +RM="rm -rf" +DIFF="diff -c" +CMP="cmp -s" +GREP='grep' +CP="cp -p" # Use -p to preserve mode,ownership,timestamps +DIRNAME='dirname' +LS='ls' +AWK='awk' + +# setup plugin path +ENVCMD="env HDF5_PLUGIN_PATH=$LD_LIBRARY_PATH/plugin" + +TESTDIR=$builddir + + +$H5CC $srcdir/h5ex_d_sz3.c -o h5ex_d_sz3 + +SRC_TESTFILES="$srcdir/testfiles" +LIST_TEST_FILES=" +$SRC_TESTFILES/h5repack_layout.h5 +$SRC_TESTFILES/h5ex_d_sz3.ddl +$SRC_TESTFILES/h5ex_d_sz3.tst +$SRC_TESTFILES/h5repack_layout.h5-ud_convert.ddl +$SRC_TESTFILES/ud_convert.h5repack_layout.h5.tst +" + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES="$LIST_TEST_FILES" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +# Print a $* message left justified in a field of 70 characters +# +MESSAGE() { + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + MESSAGE "Verifying $*" +} + +# This is different from $srcdir/../../bin/output_filter.sh +STDOUT_FILTER() { + result_file=$1 + tmp_file=/tmp/h5test_tmp_$$ + # Filter name of files. + cp $result_file $tmp_file + sed -e '/^Opening file/d' -e '/^Making file/d' \ + < $tmp_file > $result_file + # cleanup + rm -f $tmp_file +} + +# Compare the two text files +# PASS if same +# FAIL if different, and show the diff +# +# Assumed arguments: +# $1 is text file1 (expected output) +# $2 is text file2 (actual output) +CMP_OUTPUT() +{ + expect=$1 + actual=$2 + + VERIFY $@ + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " Expected Missing" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected output differs from actual output" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# ADD_H5_TEST +DUMPTEST() { + expect="$1" + actual="`basename $1 .ddl`.out" + actual_err="`basename $1 .ddl`.err" + shift + + # Run test. + TESTING $DUMPER $@ + ( + $ENVCMD $H5DUMP --enable-error-stack -p "$@" + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual + + # Clean up output file +# rm -f $actual $actual_err + +} + +REPACKTEST() +{ + infile=$1 + outfile=$2 + expect="$2.tst" + actual="$2.out1" + actual_err="$2.err1" + shift + shift + + # Run test. + TESTING H5REPACK $@ + ( + $ENVCMD $H5REPACK "$@" "out-$infile" "out-$outfile" + ) >$actual 2>$actual_err + RET=$? + STDOUT_FILTER $actual + cat $actual_err >> $actual + + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + + CMP_OUTPUT $expect $actual + fi +# rm -f $actual $actual_err +} + + +EXETEST() { + fname=$1 + expect="$2" + actual="`basename $2 .tst`.out1" + actual_err="`basename $2 .tst`.err1" + + # Run test. + TESTING $fname + ( + $ENVCMD `pwd`/$fname + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual +# rm -f $actual $actual_err +} + +################################ +######### THE TESTS ######### +################################ + +echo $ENVCMD + +COPY_TESTFILES_TO_TESTDIR + +EXETEST h5ex_d_sz3 h5ex_d_sz3.tst + +DUMPTEST h5ex_d_sz3.ddl h5ex_d_sz3.h5 + +REPACKTEST h5repack_layout.h5 out-ud_convert.h5repack_layout.h5 -v -f UD=32024,0 + +DUMPTEST h5repack_layout.h5-ud_convert.ddl -pH out-ud_convert.h5repack_layout.h5 + +#rm -f ud_convert.h5repack_layout.h5 + +echo "$nerrors tests failed in example" +exit $nerrors diff --git a/SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.ddl b/SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.ddl new file mode 100644 index 00000000..364943d6 --- /dev/null +++ b/SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.ddl @@ -0,0 +1,209 @@ +HDF5 "h5ex_d_sz3.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 4, 8 ) + SIZE 9015 (0.909:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + DATA { + (0,0): 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, + (0,15): -15, -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, + (0,27): -27, -28, -29, -30, -31, -32, -33, -34, -35, -36, -37, -38, + (0,39): -39, -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50, + (0,51): -51, -52, -53, -54, -55, -56, -57, -58, -59, -60, -61, -62, + (0,63): -63, + (1,0): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,21): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,42): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,63): 0, + (2,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + (2,18): 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + (2,34): 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + (2,50): 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + (3,0): 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, + (3,17): 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, + (3,33): 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, + (3,49): 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, + (3,62): 124, 126, + (4,0): 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, + (4,17): 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87, 90, 93, 96, + (4,33): 99, 102, 105, 108, 111, 114, 117, 120, 123, 126, 129, 132, 135, + (4,46): 138, 141, 144, 147, 150, 153, 156, 159, 162, 165, 168, 171, + (4,58): 174, 177, 180, 183, 186, 189, + (5,0): 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, + (5,17): 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, + (5,31): 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, + (5,43): 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, + (5,55): 220, 224, 228, 232, 236, 240, 244, 248, 252, + (6,0): 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, + (6,16): 80, 85, 90, 95, 100, 105, 110, 115, 120, 125, 130, 135, 140, + (6,29): 145, 150, 155, 160, 165, 170, 175, 180, 185, 190, 195, 200, + (6,41): 205, 210, 215, 220, 225, 230, 235, 240, 245, 250, 255, 260, + (6,53): 265, 270, 275, 280, 285, 290, 295, 300, 305, 310, 315, + (7,0): 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, + (7,16): 96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 162, 168, + (7,29): 174, 180, 186, 192, 198, 204, 210, 216, 222, 228, 234, 240, + (7,41): 246, 252, 258, 264, 270, 276, 282, 288, 294, 300, 306, 312, + (7,53): 318, 324, 330, 336, 342, 348, 354, 360, 366, 372, 378, + (8,0): 0, 7, 14, 21, 28, 35, 42, 49, 56, 63, 70, 77, 84, 91, 98, 105, + (8,16): 112, 119, 126, 133, 140, 147, 154, 161, 168, 175, 182, 189, + (8,28): 196, 203, 210, 217, 224, 231, 238, 245, 252, 259, 266, 273, + (8,40): 280, 287, 294, 301, 308, 315, 322, 329, 336, 343, 350, 357, + (8,52): 364, 371, 378, 385, 392, 399, 406, 413, 420, 427, 434, 441, + (9,0): 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, + (9,16): 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, + (9,28): 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, + (9,40): 320, 328, 336, 344, 352, 360, 368, 376, 384, 392, 400, 408, + (9,52): 416, 424, 432, 440, 448, 456, 464, 472, 480, 488, 496, 504, + (10,0): 0, 9, 18, 27, 36, 45, 54, 63, 72, 81, 90, 99, 108, 117, 126, + (10,15): 135, 144, 153, 162, 171, 180, 189, 198, 207, 216, 225, 234, + (10,27): 243, 252, 261, 270, 279, 288, 297, 306, 315, 324, 333, 342, + (10,39): 351, 360, 369, 378, 387, 396, 405, 414, 423, 432, 441, 450, + (10,51): 459, 468, 477, 486, 495, 504, 513, 522, 531, 540, 549, 558, + (10,63): 567, + (11,0): 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, + (11,15): 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250, 260, + (11,27): 270, 280, 290, 300, 310, 320, 330, 340, 350, 360, 370, 380, + (11,39): 390, 400, 410, 420, 430, 440, 450, 460, 470, 480, 490, 500, + (11,51): 510, 520, 530, 540, 550, 560, 570, 580, 590, 600, 610, 620, + (11,63): 630, + (12,0): 0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 110, 121, 132, 143, 154, + (12,15): 165, 176, 187, 198, 209, 220, 231, 242, 253, 264, 275, 286, + (12,27): 297, 308, 319, 330, 341, 352, 363, 374, 385, 396, 407, 418, + (12,39): 429, 440, 451, 462, 473, 484, 495, 506, 517, 528, 539, 550, + (12,51): 561, 572, 583, 594, 605, 616, 627, 638, 649, 660, 671, 682, + (12,63): 693, + (13,0): 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, + (13,14): 168, 180, 192, 204, 216, 228, 240, 252, 264, 276, 288, 300, + (13,26): 312, 324, 336, 348, 360, 372, 384, 396, 408, 420, 432, 444, + (13,38): 456, 468, 480, 492, 504, 516, 528, 540, 552, 564, 576, 588, + (13,50): 600, 612, 624, 636, 648, 660, 672, 684, 696, 708, 720, 732, + (13,62): 744, 756, + (14,0): 0, 13, 26, 39, 52, 65, 78, 91, 104, 117, 130, 143, 156, 169, + (14,14): 182, 195, 208, 221, 234, 247, 260, 273, 286, 299, 312, 325, + (14,26): 338, 351, 364, 377, 390, 403, 416, 429, 442, 455, 468, 481, + (14,38): 494, 507, 520, 533, 546, 559, 572, 585, 598, 611, 624, 637, + (14,50): 650, 663, 676, 689, 702, 715, 728, 741, 754, 767, 780, 793, + (14,62): 806, 819, + (15,0): 0, 14, 28, 42, 56, 70, 84, 98, 112, 126, 140, 154, 168, 182, + (15,14): 196, 210, 224, 238, 252, 266, 280, 294, 308, 322, 336, 350, + (15,26): 364, 378, 392, 406, 420, 434, 448, 462, 476, 490, 504, 518, + (15,38): 532, 546, 560, 574, 588, 602, 616, 630, 644, 658, 672, 686, + (15,50): 700, 714, 728, 742, 756, 770, 784, 798, 812, 826, 840, 854, + (15,62): 868, 882, + (16,0): 0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, + (16,14): 210, 225, 240, 255, 270, 285, 300, 315, 330, 345, 360, 375, + (16,26): 390, 405, 420, 435, 450, 465, 480, 495, 510, 525, 540, 555, + (16,38): 570, 585, 600, 615, 630, 645, 660, 675, 690, 705, 720, 735, + (16,50): 750, 765, 780, 795, 810, 825, 840, 855, 870, 885, 900, 915, + (16,62): 930, 945, + (17,0): 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, + (17,14): 224, 240, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, + (17,26): 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, + (17,38): 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, + (17,50): 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, + (17,62): 992, 1008, + (18,0): 0, 17, 34, 51, 68, 85, 102, 119, 136, 153, 170, 187, 204, 221, + (18,14): 238, 255, 272, 289, 306, 323, 340, 357, 374, 391, 408, 425, + (18,26): 442, 459, 476, 493, 510, 527, 544, 561, 578, 595, 612, 629, + (18,38): 646, 663, 680, 697, 714, 731, 748, 765, 782, 799, 816, 833, + (18,50): 850, 867, 884, 901, 918, 935, 952, 969, 986, 1003, 1020, 1037, + (18,62): 1054, 1071, + (19,0): 0, 18, 36, 54, 72, 90, 108, 126, 144, 162, 180, 198, 216, 234, + (19,14): 252, 270, 288, 306, 324, 342, 360, 378, 396, 414, 432, 450, + (19,26): 468, 486, 504, 522, 540, 558, 576, 594, 612, 630, 648, 666, + (19,38): 684, 702, 720, 738, 756, 774, 792, 810, 828, 846, 864, 882, + (19,50): 900, 918, 936, 954, 972, 990, 1008, 1026, 1044, 1062, 1080, + (19,61): 1098, 1116, 1134, + (20,0): 0, 19, 38, 57, 76, 95, 114, 133, 152, 171, 190, 209, 228, 247, + (20,14): 266, 285, 304, 323, 342, 361, 380, 399, 418, 437, 456, 475, + (20,26): 494, 513, 532, 551, 570, 589, 608, 627, 646, 665, 684, 703, + (20,38): 722, 741, 760, 779, 798, 817, 836, 855, 874, 893, 912, 931, + (20,50): 950, 969, 988, 1007, 1026, 1045, 1064, 1083, 1102, 1121, 1140, + (20,61): 1159, 1178, 1197, + (21,0): 0, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200, 220, 240, 260, + (21,14): 280, 300, 320, 340, 360, 380, 400, 420, 440, 460, 480, 500, + (21,26): 520, 540, 560, 580, 600, 620, 640, 660, 680, 700, 720, 740, + (21,38): 760, 780, 800, 820, 840, 860, 880, 900, 920, 940, 960, 980, + (21,50): 1000, 1020, 1040, 1060, 1080, 1100, 1120, 1140, 1160, 1180, + (21,60): 1200, 1220, 1240, 1260, + (22,0): 0, 21, 42, 63, 84, 105, 126, 147, 168, 189, 210, 231, 252, 273, + (22,14): 294, 315, 336, 357, 378, 399, 420, 441, 462, 483, 504, 525, + (22,26): 546, 567, 588, 609, 630, 651, 672, 693, 714, 735, 756, 777, + (22,38): 798, 819, 840, 861, 882, 903, 924, 945, 966, 987, 1008, 1029, + (22,50): 1050, 1071, 1092, 1113, 1134, 1155, 1176, 1197, 1218, 1239, + (22,60): 1260, 1281, 1302, 1323, + (23,0): 0, 22, 44, 66, 88, 110, 132, 154, 176, 198, 220, 242, 264, 286, + (23,14): 308, 330, 352, 374, 396, 418, 440, 462, 484, 506, 528, 550, + (23,26): 572, 594, 616, 638, 660, 682, 704, 726, 748, 770, 792, 814, + (23,38): 836, 858, 880, 902, 924, 946, 968, 990, 1012, 1034, 1056, + (23,49): 1078, 1100, 1122, 1144, 1166, 1188, 1210, 1232, 1254, 1276, + (23,59): 1298, 1320, 1342, 1364, 1386, + (24,0): 0, 23, 46, 69, 92, 115, 138, 161, 184, 207, 230, 253, 276, 299, + (24,14): 322, 345, 368, 391, 414, 437, 460, 483, 506, 529, 552, 575, + (24,26): 598, 621, 644, 667, 690, 713, 736, 759, 782, 805, 828, 851, + (24,38): 874, 897, 920, 943, 966, 989, 1012, 1035, 1058, 1081, 1104, + (24,49): 1127, 1150, 1173, 1196, 1219, 1242, 1265, 1288, 1311, 1334, + (24,59): 1357, 1380, 1403, 1426, 1449, + (25,0): 0, 24, 48, 72, 96, 120, 144, 168, 192, 216, 240, 264, 288, 312, + (25,14): 336, 360, 384, 408, 432, 456, 480, 504, 528, 552, 576, 600, + (25,26): 624, 648, 672, 696, 720, 744, 768, 792, 816, 840, 864, 888, + (25,38): 912, 936, 960, 984, 1008, 1032, 1056, 1080, 1104, 1128, 1152, + (25,49): 1176, 1200, 1224, 1248, 1272, 1296, 1320, 1344, 1368, 1392, + (25,59): 1416, 1440, 1464, 1488, 1512, + (26,0): 0, 25, 50, 75, 100, 125, 150, 175, 200, 225, 250, 275, 300, + (26,13): 325, 350, 375, 400, 425, 450, 475, 500, 525, 550, 575, 600, + (26,25): 625, 650, 675, 700, 725, 750, 775, 800, 825, 850, 875, 900, + (26,37): 925, 950, 975, 1000, 1025, 1050, 1075, 1100, 1125, 1150, 1175, + (26,48): 1200, 1225, 1250, 1275, 1300, 1325, 1350, 1375, 1400, 1425, + (26,58): 1450, 1475, 1500, 1525, 1550, 1575, + (27,0): 0, 26, 52, 78, 104, 130, 156, 182, 208, 234, 260, 286, 312, + (27,13): 338, 364, 390, 416, 442, 468, 494, 520, 546, 572, 598, 624, + (27,25): 650, 676, 702, 728, 754, 780, 806, 832, 858, 884, 910, 936, + (27,37): 962, 988, 1014, 1040, 1066, 1092, 1118, 1144, 1170, 1196, + (27,47): 1222, 1248, 1274, 1300, 1326, 1352, 1378, 1404, 1430, 1456, + (27,57): 1482, 1508, 1534, 1560, 1586, 1612, 1638, + (28,0): 0, 27, 54, 81, 108, 135, 162, 189, 216, 243, 270, 297, 324, + (28,13): 351, 378, 405, 432, 459, 486, 513, 540, 567, 594, 621, 648, + (28,25): 675, 702, 729, 756, 783, 810, 837, 864, 891, 918, 945, 972, + (28,37): 999, 1026, 1053, 1080, 1107, 1134, 1161, 1188, 1215, 1242, + (28,47): 1269, 1296, 1323, 1350, 1377, 1404, 1431, 1458, 1485, 1512, + (28,57): 1539, 1566, 1593, 1620, 1647, 1674, 1701, + (29,0): 0, 28, 56, 84, 112, 140, 168, 196, 224, 252, 280, 308, 336, + (29,13): 364, 392, 420, 448, 476, 504, 532, 560, 588, 616, 644, 672, + (29,25): 700, 728, 756, 784, 812, 840, 868, 896, 924, 952, 980, 1008, + (29,37): 1036, 1064, 1092, 1120, 1148, 1176, 1204, 1232, 1260, 1288, + (29,47): 1316, 1344, 1372, 1400, 1428, 1456, 1484, 1512, 1540, 1568, + (29,57): 1596, 1624, 1652, 1680, 1708, 1736, 1764, + (30,0): 0, 29, 58, 87, 116, 145, 174, 203, 232, 261, 290, 319, 348, + (30,13): 377, 406, 435, 464, 493, 522, 551, 580, 609, 638, 667, 696, + (30,25): 725, 754, 783, 812, 841, 870, 899, 928, 957, 986, 1015, 1044, + (30,37): 1073, 1102, 1131, 1160, 1189, 1218, 1247, 1276, 1305, 1334, + (30,47): 1363, 1392, 1421, 1450, 1479, 1508, 1537, 1566, 1595, 1624, + (30,57): 1653, 1682, 1711, 1740, 1769, 1798, 1827, + (31,0): 0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, + (31,13): 390, 420, 450, 480, 510, 540, 570, 600, 630, 660, 690, 720, + (31,25): 750, 780, 810, 840, 870, 900, 930, 960, 990, 1020, 1050, 1080, + (31,37): 1110, 1140, 1170, 1200, 1230, 1260, 1290, 1320, 1350, 1380, + (31,47): 1410, 1440, 1470, 1500, 1530, 1560, 1590, 1620, 1650, 1680, + (31,57): 1710, 1740, 1770, 1800, 1830, 1860, 1890 + } + } +} +} diff --git a/SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.tst b/SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.tst new file mode 100644 index 00000000..d3622b4c --- /dev/null +++ b/SZ3/config/cmake/binex/example/testfiles/h5ex_d_sz3.tst @@ -0,0 +1,12 @@ +sz3 filter is available for encoding and decoding. +....Create dataset ................ +....Writing sz3 compressed data ................ +....Close the file and reopen for reading ........ +Filter info is available from the dataset creation property + Filter identifier is 32024 + Number of parameters is 10 with the value 50593792 + To find more about the filter check H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 +....Reading sz3 compressed data ................ +Maximum value in DS1 is 1890.00 +Maximum absolute error is within the error bound 0.001 +sz3 filter is available now since H5Dread triggered loading of the filter. diff --git a/SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5 b/SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2cc44b1405e955fde6483ccb2ddf2268b0f1d4de GIT binary patch literal 27280 zcmeI)1=Lm5+9=?U5=2o1R7_A5P*6z=>F)0C?(XjH?(XjH?(XjHhWl*!IQJZXk8;Po zXZ+*d%VIox)m(e8Z>~A#dfzqJyJLnl=^}*xIDAlG3KuRYbWp_K7hk?S51x8dda>o@ z`pf+#f)_#r&pQOqCxm>l5F#k>FDNK#@b>qD-~W&0^5jaJHb}{r@BeZhd|1$gP%oA; zyxj15{{K}CWJsGbN8k@G5d2raY;PNa+u6&?C{w#^o%mJ)^8~^3gu(Mf!Smv!E7Yx4 z*~&lGt5>a2vUHt)TCY~8V)?q&>(%wN1l@;QMiP$o>~!fA77 z%bq4gP}tzDJ|(z6{9fo!US9U%8!t}kPfEPl^s-+z2=i}?{Jtal%iV9hT=?_(%fGY3 z-<10~`0%JX|NMPkZp{$95IyY6E&qG-|NQkl5qydN`g;E7&!fZ_FKX()`#es3_rG!; z9fB`!h@clODa4Cz7QFU9`_IdE_VPaQ@A~0C`fI2cZ}g9T7cx|dUtfGy;Ccn_f4}#X zf9`h)lSIq*pTABKi@i9FzrIc{%KYE?8W_U@!(d>@3RH4ns0j=Yfi4}6Hwj#Tw+IY^ z5qO7p35;X!5sAnIF4p^ez=s6JuaEecPxzG2_?&1&C(seTBnDscH8F`rpdEd~x5ObX zfd>0MKM;?={UmTN3EW2#k(eL(iNHAa3xT$jgx^R?GLn;ml%ygxX%OfHr6WBV$VetK zlZC8gBRhecUSNpKMQ-wtmwe=>00k*TVTw?cK%**72})9m(v+brY(34*D zrVoATM}Gz|kUW_xyE&FaFbiy<_>qc$9*2~kVib`2~T;(a{>WO2m+&T zC_)p4Hwa5O-XuJ4@iq~7hj)p{dqg5KQFxyZ_>ic4#K(NXr+mieL?b$1@Fg+$im!=D zEMoHw-x7zoe8>0vKs@4;fP^F>F+cJXKl2N}l7!z#N-~m@f|R5pHG;UHw4@_F8OTT` zGLwa@xbXyKCkHvnMQ-wtmwe=>00k*TVTw?cViczYB`HN|%21Yal&1m}sYGR}P?c&_ zrv^2tMQ!R(mwMEv0S#$HW17&EW;CY-Eont-+R&DEw5J0d=|pF`(3Ngq#cl3zmwVjj0S|e^W1jGoXFMlx;}7IOAqhoj z!te%R3CEj+=PlkQ0`KrH5qXbDL?#OF^8p_cm5=zCPxzG2_?&1&=L^0h24C?tF^NTN zzTsQq5SQ=xo*#%ud=ik5L?q@%e&T0-;a8II8%aqWF`w) z$wqc^kds{GCJ%YZM}7)WkU|uu2t_GIaY|5Y(34*DrVoATM}Gz| zkUW_xyE&FaFbiy<_>qc$9*2~kVib`2~T;(a{@ts2tpEy(1hU)!V-=* z3C~-+O$6TIT_W-xk%&wb-sb~8Bq|^AF`w`$pYb`-2u#7lP~cQv=O5}IE$K*41~QU~ z%w!=e*~m@~a*~VODP6Jl zYEp~Z)S)i*s80hL(ul@1p()L1P77Mniq^EDE$wJe2RhP;&UB$G-RMpadeV#D^r0{P z=+6KKGKj$pVJO2G&Im>_iqVW=EaMo@1ST?x$xLA?)0oZ-W-^P}%waC`n9l+hvWUej zVJXX4&I(qtiq))PE$dj%1~#&Z&1_*S+t|(ycCw4z>|rna*v|nDa)`qm;V8#A&IwL( ziqo9oEay1S1uk-l%Ut0q*SO9NZgPv;+~F?wxX%L~@`%Sg;VI8}PN4ILAS9s(O&H!F zEa7;Q@Vv#_MBp9XB_i(;iO59ZeLmnrqVf?R^9i5w8J`o4=zPJK#NaEwCML0n%{P2Y z9OCjF-}3|Uh))6%l8D6o$WQ#tFZ@apej_Q#NKOh;l8V%%AuZ`hPX;oQiOggnE7{0S z4sw!<+~grI`N&TJ3Q~x|6rm`^C{77VQi{@)p)BPnPX#JciON)=D%Ge?4Qf)0+SH*g z^{7t+8q$cyG@&WYXif`S(u&r!p)KubPX{{EiOzJPE8XZ$4|>vz-t?g_{pimC1~Q1j z3}Gn47|sYrGK$fRVJzbq&jcniiOEc1D$|(G3}!Nm+00=s^O(;97P5%NEMY0jSk4Mo zvWnHLVJ+)e&jvQKiOph4=Y@4~fc0e9R|&%4d8|G@|nbUlN0__?no+A~xUfEpdp;cYMze z#3MclNJt_Q^CLg;Gr#aFN%)PVBqKQ~NJ%PElZLdUBRv_&NG39qg{)*FJ2}WnE^?EH zyyPQ41t>@%3R8rl6r(sLC`l}a>$Rs8+g{e$qIy0EbEM_x@xy)le3s}e^7PEw- zV?7(#$R;+kg{^F3J3H9PE_Snrz3gK@2RO(f4s(Q~9OF1AILRqabB42=<2)C*$R#dw zg{xfSIybnlYxw6A~RXYN;a~SgPi0d zH+jfQKJrt5f)t`KMJP%!ic^A;l%h0cC`&oYQ-O+9qB2#eN;RregPPQ$Hg%{=J?hhd zhBTrvO=wCpn$v=ow4ya_XiGcV(}9k3qBC9SN;kUGgP!!FH+|?!Kl(F(fed0WLm0|1 zhBJbZjAArn7|S@uGl7XrVlq>h$~2}kgPF`?HglNEJm#~2g)Cw*OIXS>ma~GDtYS55 zSj#%rvw@9lVl!LV$~LyMgPrVRH+$I2KK65fgB;>8M>xtcj&p*OoZ>WRILkTCbAgLo z;xbpb$~CTYgPYvqHg~woJ?`^>hdkmjPk72Ro)Z+}b^ei39i$>PX-G>t(vyLVWFj+J z$VxV{lY^Y(A~$)+OFr^bfPxgFFhwXzF^W@yl9Zw}WhhHI%2R=gRH8Cfs7f`eQ-hk+ zqBeD?OFin-fQB@pF->SnGn&(amb9WZZD>n7+S7rKbfPm|=t?)b(}SM$qBni$OF#NE zfPoBRFhdy1ForXNk&I$AV;IXg#xsG5Oky%qn94M!GlQATVm5P_%RJ_@fQ2k#F-us= zGM2M~m8@blYgo%V*0X_)Y+^H8*vdAxvxA-NVmEu(%RcsVfP)<3Fh@AbF^+SBlbqr- zXE@6_&U1l_T;eiUxXLxIbAy}Q;x>1<%RTP%fQLNdF;95PGoBL^@^$`^QXQlsHEBpo zI?|JYjASA+S;$H@vXg_HI4f|8V?G-W7DIm%Okid3R9 zRj5ies#AlS)S@Q6 z^rAO?=u1EPGk}2%VlYD($}omAf{~13G-DXcIL0%9iA-WLQ<%y$rZa|!^2*vmflbAW>!;xI=z z$}x^}f|H!$G-o)=InHx|i(KL|SGdYGu5*K%+~PKOxXV56^MHpu;xSKn$}^txI{)ze zLp6L!48G!PViJqke8acIAuiwXJwFhS_#_}9iAc>6Q1&n=LCg# zoqwcM2dPL+8q$)E^kg6-naE5QvXYJLHNAm8eV=s#1;W)SxD{s7)Q}QjhvHpdpQDOcR>YjOMhUC9P;p8`{#2_H>{l zo#;##y3&pA^q?ob=uIE`(vSWOU?77S%n*h$jNy!6B%>J37{)S=@l0SMlbFmDrZSD` z%wQ(7n9UsKGLQKzU?GcG%o3KejODCgC97D?8rHIo^=x1xo7l`2wz7@w>|iIm*v%gH zvXA{7;2?)M%n^=qjN_c(B&Rsd8P0N!^IYH}m$=Lou5yj*+~6j+xXm5za*z8w;31EA z%oCpSjOPS}e4T%!R0pX@O&ZdYj`U<8Bbmrd7P69!?BpOPxyVf(@{*7I6rdo5C`=KG zQjFr1pd_UzO&Q8kj`CEXB9*926{=E=>eQenwWv)U>QayTG@v1kXiO8D(v0S`pe3zn zO&i+Mj`nn*Bc13>7rN4o?)0E1z35FJ`qGd73}7IG7|alcGK}GjU?ig$%^1cqj`2)j zB9oZR6s9tb>C9jzvzW~s<}#1@EMOsvSj-ZZvW(@dU?r)hZbx46w6 z?sAX&Jm4XZc+3-?@{H&Fcjq60?{$U<3jJTc*BL!jAUOH=(||ND_WiM3_&*%7ApO6V zi}Hv4FINv_`tx#se7^@-Umo#d{__1^mJ9u_yI-%pu7TGz@VW+G*TCxR!ipgMzdha{Ea7;QKz#N-AMhbj`G}ACgira5*nGpc#33%<@jX8fkNEsX zQj(FJ6r>~-sR@LoX~{uOa*>-n zNkn2&kdjoSCJhpq|MJ{0k8OF$M}7)WkU|uu2t}ztMJiF5DpaK!)u};En$VPHG^YhE zX+>+=(3U_x@)mCsfp>V9h`dK6BJ(-Xh|U*$NesT?Yhn_M1SBL8iTRPA_?ch$l_aDi zJsHSICNh(StYjlQg(yrBic*Z?l%OP~C`~o0Q-hk+qBeD?OFin-iq^EDE$wJe2RhP; z&h(={0~p941~Y`A3}ZNxn9LNWGL7lXU?#Je%`%p=f|aadHEUSQI@Ys`-Rxm6``FI` z4swXYoZ~zfxX2|gbA_v1<2sLc%oCpSjOPS}_}Az8?^u7Ah`dK6A`^x8`G61kk{Eo& z*Tf_ivH6B?iNlZl#LxV~uO#6&l9G(%WFj+J$VxV{lY^Y(A~(e-P6>(8$u4%YhrOKOB&Rsd8P0N!^IYH}ceu+v?(=|$ zJmN7=cuKgCulE1gKUMBCJ|`N{`GPNr!B>1uJmQmpgd`#{Kk^el^9yN6OFGh%fsAA# zGg-(=0SZ!x!W5w>#VAe*N>YWYRHHgIs7WnqQ-`{=pe3znO&i+Mj`nn*BYo&gKl(F( zfed0WLm0{gCh~XBA(tq#lw~Yu1uI#_YSyrpSAA}HM%lCc-Sfa#ea`o)-=7?$%?xG; zLm9?!Mlh05jAk0snZZnEF`GHeWghcc#cI~DmUXOW0~^`IX7;h4103WKhdIJgj&Ynz zT;>W_xyE&FaFbiy<{8fkQimZ3Nhm@ShBpXHBq9@q_xXSiiONTO%qPSo7P0w;Z;3-( zzT-n*a)9`||G?-#%7_l95o?$d=p zfMxo}???szAl1M7y{W(Z-M4@KeW}#tqGbyg)ZzC}2o?PMZ^2FZFLR#o&gU%x2mV#; z7sdYf`G60J%13<6Cw$6hd`>i?^95fLgFnWIv2FW?Z;3-(zTi8*s9my=xNCJ%YZM}7)WkU|uu2t_GIaY|5AZhTiM2TcCeFIjm3{Bdz51w=L9D?#c9rPmUEov0vEZ&Wv+0QYh33BH@U@a?r@iT z+~)xgdBkI$@RVmfCn&_f#`dADhv5yv5{@?s&s)4r1m59YBJv)Qh)fjT=L0?@Dj)GN zpYSQ4@j20m&KG=148G!PViJqke8acIAuiwXJwFhS_#_}9iAc> z6Q1&n=LCiP>liki{cjSUw|JWfyu-Uh1uOk(kO zF>yR);*)@cBqA|C@)JMv3$GeGr%^U7=}1ooGLnhRWFafr$W9J&l8fBr;h$sd0@e#s zh{6=1D8(pF2})9m(v+br{luNvF;QMNDr=+6KKGKj$pVJNSf158kMB9oZR6s9tb>C9jzvzW~s<}!~z%N3T` zzLaGwX9X)+#cI~DmUXOW0~^`IpXC%gY~RT)cC&}Q>|;L%ILNE!9%qz2%Q?<-fs0(? zGFQ0DHLi1mo800yceu+v?(=|$JmN7=c*--L6A1c4@T$4YtLFNzn)3ugkiZl?6a`Kv zC^TVsgRq3-O~UgQZxexcc$bL0Mh4=Y@*Ez`R9AtoYFpxnEW(Y$W#&AY3l2MH2 zbq?}62l?876q8uQ<{Q2x4srR8@A-jv#3um>Nkn3P+=(3WeG#AU8h4=Y@4~fc0e9R|&%4dAe>m1~D4l+VJ W7|AF`GlsE@V?3{OkpH)HkpBU1R2eV; literal 0 HcmV?d00001 diff --git a/SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl new file mode 100644 index 00000000..2fced728 --- /dev/null +++ b/SZ3/config/cmake/binex/example/testfiles/h5repack_layout.h5-ud_convert.ddl @@ -0,0 +1,158 @@ +HDF5 "out-ud_convert.h5repack_layout.h5" { +GROUP "/" { + DATASET "dset1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset2" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset3" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset4" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_chunk" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } + STORAGE_LAYOUT { + CHUNKED ( 20, 10 ) + SIZE 552 (5.797:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_compact" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_contiguous" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/SZ3/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst b/SZ3/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst new file mode 100644 index 00000000..64e56c1d --- /dev/null +++ b/SZ3/config/cmake/binex/example/testfiles/ud_convert.h5repack_layout.h5.tst @@ -0,0 +1,15 @@ +No all objects to modify layout +All objects to apply filter are... + User Defined 32024 +Making new file ... +----------------------------------------- + Type Filter (Compression) Name +----------------------------------------- + group / + dset UD (23.704:1) /dset1 + dset UD (23.704:1) /dset2 + dset UD (23.704:1) /dset3 + dset UD (23.704:1) /dset4 + dset UD (5.797:1) /dset_chunk + dset UD (23.704:1) /dset_compact + dset UD (23.704:1) /dset_contiguous diff --git a/SZ3/config/cmake/cacheinit.cmake b/SZ3/config/cmake/cacheinit.cmake new file mode 100644 index 00000000..c5a6854f --- /dev/null +++ b/SZ3/config/cmake/cacheinit.cmake @@ -0,0 +1,29 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# This is the CMakeCache file. + +######################## +# EXTERNAL cache entries +######################## + +set (H5PL_BUILD_TESTING ON CACHE BOOL "Build h5sz3 Unit Testing" FORCE) + +set (H5PL_BUILD_EXAMPLES ON CACHE BOOL "Build h5sz3 Examples" FORCE) + +set (HDF5_PACKAGE_NAME "hdf5" CACHE STRING "Name of HDF5 package" FORCE) + +set (HDF5_NAMESPACE "hdf5::" CACHE STRING "Name space of HDF5 library" FORCE) + +set (H5SZ3_CPACK_ENABLE ON CACHE BOOL "Enable the CPACK include and components" FORCE) + +set (H5PL_ALLOW_EXTERNAL_SUPPORT "NO" CACHE STRING "Allow External Library Building (NO GIT TGZ)" FORCE) +set_property (CACHE H5PL_ALLOW_EXTERNAL_SUPPORT PROPERTY STRINGS NO GIT TGZ) diff --git a/SZ3/config/cmake/config.h.in b/SZ3/config/cmake/config.h.in new file mode 100644 index 00000000..46057076 --- /dev/null +++ b/SZ3/config/cmake/config.h.in @@ -0,0 +1,91 @@ +/* config.h.in. Generated By CMake during the configuration */ + +#ifndef CONFIG_H_ +#define CONFIG_H_ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_DLFCN_H @HAVE_DLFCN_H@ + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +#cmakedefine HAVE_DOPRNT @HAVE_DOPRNT@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_FCNTL_H @HAVE_FCNTL_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_INTTYPES_H @HAVE_INTTYPES_H@ + +/* Define to 1 if you have the `sz3' library (-lsz3). */ +#cmakedefine HAVE_LIBSZ3 @HAVE_LIBSZ3@ + +/* Define to 1 if you have the `hdf5' library (-lhdf5). */ +#cmakedefine HAVE_LIBHDF5 @HAVE_LIBHDF5@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_MEMORY_H @HAVE_MEMORY_H@ + +/* Define to 1 if you have the `memset' function. */ +#cmakedefine HAVE_MEMSET @HAVE_MEMSET@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STDINT_H @HAVE_STDINT_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STDLIB_H @HAVE_STDLIB_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STRINGS_H @HAVE_STRINGS_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_STRING_H @HAVE_STRING_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_STAT_H ${HAVE_SYS_STAT_H} + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_TYPES_H ${HAVE_SYS_TYPES_H} + +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_UNISTD_H @HAVE_UNISTD_H@ + +/* Define to 1 if you have the `vprintf' function. */ +#cmakedefine HAVE_VPRINTF @HAVE_VPRINTF@ + +/* Define to the sub-directory in which libtool stores uninstalled libraries. + */ +#define LT_OBJDIR "@LT_OBJDIR@" + +/* Name of package */ +#define PACKAGE "@H5SZ3_PACKAGE@" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "@H5SZ3_PACKAGE_BUGREPORT@" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "@H5SZ3_PACKAGE_NAME@" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "@H5SZ3_PACKAGE_STRING@" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "@H5SZ3_PACKAGE_TARNAME@" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "@H5SZ3_PACKAGE_URL@" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "@H5SZ3_PACKAGE_VERSION@" + +/* Define to 1 if you have the ANSI C header files. */ +#cmakedefine STDC_HEADERS "@STDC_HEADERS@" + +/* Version number of package */ +#define VERSION "@H5SZ3_VERSION@" + +/* Define to empty if `const' does not conform to ANSI C. */ +#cmakedefine const + +/* Define to `unsigned int' if does not define. */ +#cmakedefine size_t + +#endif diff --git a/SZ3/config/cmake/distribution.entitlements b/SZ3/config/cmake/distribution.entitlements new file mode 100644 index 00000000..0e0df6c7 --- /dev/null +++ b/SZ3/config/cmake/distribution.entitlements @@ -0,0 +1,16 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.disable-executable-page-protection + + com.apple.security.cs.disable-library-validation + + com.apple.security.cs.allow-dyld-environment-variables + + + diff --git a/SZ3/config/cmake/grepTest.cmake b/SZ3/config/cmake/grepTest.cmake new file mode 100644 index 00000000..2c83b3c8 --- /dev/null +++ b/SZ3/config/cmake/grepTest.cmake @@ -0,0 +1,199 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# grepTest.cmake executes a command and captures the output in a file. File is then compared +# against a reference file. Exit status of command can also be compared. +cmake_policy(SET CMP0007 NEW) + +# arguments checking +if (NOT TEST_PROGRAM) + message (FATAL_ERROR "Require TEST_PROGRAM to be defined") +endif () +if (NOT TEST_FOLDER) + message (FATAL_ERROR "Require TEST_FOLDER to be defined") +endif () +if (NOT TEST_OUTPUT) + message (FATAL_ERROR "Require TEST_OUTPUT to be defined") +endif () +if (NOT TEST_FILTER) + message (STATUS "Optional TEST_FILTER to be defined") +endif () +if (NOT TEST_REFERENCE) + message (FATAL_ERROR "Require TEST_REFERENCE to be defined") +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}) +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err) +endif () + +message (STATUS "COMMAND: ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS}") + +if (TEST_LIBRARY_DIRECTORY) + if (WIN32) + set (ENV{PATH} "$ENV{PATH};${TEST_LIBRARY_DIRECTORY}") + elseif (APPLE) + set (ENV{DYLD_LIBRARY_PATH} "$ENV{DYLD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + else () + set (ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + endif () + set (ENV{HDF5_PLUGIN_PATH} "${TEST_LIBRARY_DIRECTORY}/plugins:${TEST_LIBRARY_DIRECTORY}/plugin:$ENV{HDF5_PLUGIN_PATH}") +endif () + +if (TEST_ENV_VAR) + set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}") + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (TRACE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}") + endif () +endif () + +# run the test program, capture the stdout/stderr and the result var +execute_process ( + COMMAND ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS} + WORKING_DIRECTORY ${TEST_FOLDER} + RESULT_VARIABLE TEST_RESULT + OUTPUT_FILE ${TEST_OUTPUT} + ERROR_FILE ${TEST_OUTPUT}.err + OUTPUT_VARIABLE TEST_OUT + ERROR_VARIABLE TEST_ERROR +) + +message (STATUS "COMMAND Result: ${TEST_RESULT}") + +message (STATUS "COMMAND Error: ${TEST_ERROR}") + +# remove special output +file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) +string (FIND TEST_STREAM "_pmi_alps" TEST_FIND_RESULT) +if (TEST_FIND_RESULT GREATER -1) + string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_STREAM}) +endif () + +# if the TEST_ERRREF exists grep the error output with the error reference +set (TEST_ERRREF_RESULT 0) +if (TEST_ERRREF) + # if the .err file exists grep the error output with the error reference before comparing stdout + if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_ERR_STREAM) + list (LENGTH TEST_ERR_STREAM test_len) + if (test_len GREATER 0) + # TEST_ERRREF should always be matched + string (REGEX MATCH "${TEST_ERRREF}" TEST_MATCH ${TEST_ERR_STREAM}) + string (COMPARE EQUAL "${TEST_ERRREF}" "${TEST_MATCH}" TEST_ERRREF_RESULT) + if (NOT TEST_ERRREF_RESULT) + message (FATAL_ERROR "Failed: The error output of ${TEST_PROGRAM} did not contain ${TEST_ERRREF}") + endif () + endif () + endif () + + #always compare output file to reference unless this must be skipped + set (TEST_COMPARE_RESULT 0) + if (NOT TEST_SKIP_COMPARE) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}") + file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + if (WIN32) + configure_file(${TEST_FOLDER}/${TEST_REFERENCE} ${TEST_FOLDER}/${TEST_REFERENCE}.tmp NEWLINE_STYLE CRLF) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}.tmp") + file(RENAME ${TEST_FOLDER}/${TEST_REFERENCE}.tmp ${TEST_FOLDER}/${TEST_REFERENCE}) + endif () + #file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + #file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") + endif () + if (NOT TEST_SORT_COMPARE) + # now compare the output with the reference + execute_process ( + COMMAND ${CMAKE_COMMAND} -E compare_files ${CMAKE_IGNORE_EOL} ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE} + RESULT_VARIABLE TEST_COMPARE_RESULT + ) + else () + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2) + list (SORT v1) + list (SORT v2) + if (NOT v1 STREQUAL v2) + set (TEST_COMPARE_RESULT 1) + endif () + endif () + + if (TEST_COMPARE_RESULT) + set (TEST_COMPARE_RESULT 0) + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act) + list (LENGTH test_act len_act) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref) + list (LENGTH test_ref len_ref) + if (len_act GREATER 0 AND len_ref GREATER 0) + math (EXPR _FP_LEN "${len_ref} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + list (GET test_act ${line} str_act) + list (GET test_ref ${line} str_ref) + if (NOT str_act STREQUAL str_ref) + if (str_act) + set (TEST_COMPARE_RESULT 1) + message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") + endif () + endif () + endforeach () + else () + if (len_act EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty") + endif () + if (len_ref EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty") + endif () + endif () + if (NOT len_act EQUAL len_ref) + set (TEST_COMPARE_RESULT 1) + endif () + endif () + endif () + + message (STATUS "COMPARE Result: ${TEST_COMPARE_RESULT}") + + # again, if return value is !=0 scream and shout + if (TEST_COMPARE_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}") + endif () + endif () + endif () +else () + # else grep the output with the reference + set (TEST_GREP_RESULT 0) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + + # TEST_REFERENCE should always be matched + string (REGEX MATCH "${TEST_REFERENCE}" TEST_MATCH ${TEST_STREAM}) + string (COMPARE EQUAL "${TEST_REFERENCE}" "${TEST_MATCH}" TEST_GREP_RESULT) + if (NOT TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}") + endif () +endif () + + +if (TEST_FILTER) + string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) + if (TEST_EXPECT) + # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match + string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) + if (TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") + endif () + endif () +endif () + +# everything went fine... +message (STATUS "Passed: The output of ${TEST_PROGRAM} matched") + diff --git a/SZ3/config/cmake/h5sz3-config-version.cmake.in b/SZ3/config/cmake/h5sz3-config-version.cmake.in new file mode 100644 index 00000000..6c7a2343 --- /dev/null +++ b/SZ3/config/cmake/h5sz3-config-version.cmake.in @@ -0,0 +1,38 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +#----------------------------------------------------------------------------- +# h5sz3 Version file for install directory +#----------------------------------------------------------------------------- + +set (PACKAGE_VERSION @H5SZ3_VERSION_STRING@) + +if ("${PACKAGE_FIND_VERSION_MAJOR}" EQUAL @H5SZ3_VERSION_MAJOR@) + + # exact match for version @H5SZ3_VERSION_MAJOR@.@H5SZ3_VERSION_MINOR@ + if ("${PACKAGE_FIND_VERSION_MINOR}" EQUAL @H5SZ3_VERSION_MINOR@) + + # compatible with any version @H5SZ3_VERSION_MAJOR@.@H5SZ3_VERSION_MINOR@.x + set (PACKAGE_VERSION_COMPATIBLE 1) + + if ("${PACKAGE_FIND_VERSION_PATCH}" EQUAL @H5SZ3_VERSION_RELEASE@) + set (PACKAGE_VERSION_EXACT 1) + + if ("${PACKAGE_FIND_VERSION_TWEAK}" EQUAL @H5SZ3_VERSION_SUBRELEASE@) + # not using this yet + endif () + + endif () + + endif () +endif () + + diff --git a/SZ3/config/cmake/h5sz3-config.cmake.in b/SZ3/config/cmake/h5sz3-config.cmake.in new file mode 100644 index 00000000..f751f087 --- /dev/null +++ b/SZ3/config/cmake/h5sz3-config.cmake.in @@ -0,0 +1,70 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +#----------------------------------------------------------------------------- +# h5sz3 Config file for compiling against sz3 install directory +#----------------------------------------------------------------------------- +@PACKAGE_INIT@ + +string(TOUPPER @H5SZ3_PACKAGE@ H5SZ3_PACKAGE_NAME) + +set (${H5SZ3_PACKAGE_NAME}_VALID_COMPONENTS static shared) + +#----------------------------------------------------------------------------- +# User Options +#----------------------------------------------------------------------------- +set (${H5SZ3_PACKAGE_NAME}_BUILD_SHARED_LIBS @BUILD_SHARED_LIBS@) +set (${H5SZ3_PACKAGE_NAME}_EXPORT_LIBRARIES @H5SZ3_LIBRARIES_TO_EXPORT@) + +#----------------------------------------------------------------------------- +# Directories +#----------------------------------------------------------------------------- +set (${H5SZ3_PACKAGE_NAME}_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@") + +set (${H5SZ3_PACKAGE_NAME}_SHARE_DIR "@PACKAGE_SHARE_INSTALL_DIR@") +set_and_check (${H5SZ3_PACKAGE_NAME}_BUILD_DIR "@PACKAGE_CURRENT_BUILD_DIR@") + +#----------------------------------------------------------------------------- +# Version Strings +#----------------------------------------------------------------------------- +set (${H5SZ3_PACKAGE_NAME}_VERSION_STRING @H5SZ3_VERSION_STRING@) +set (${H5SZ3_PACKAGE_NAME}_VERSION_MAJOR @H5SZ3_VERSION_MAJOR@) +set (${H5SZ3_PACKAGE_NAME}_VERSION_MINOR @H5SZ3_VERSION_MINOR@) + +#----------------------------------------------------------------------------- +# Don't include targets if this file is being picked up by another +# project which has already build h5sz3 as a subproject +#----------------------------------------------------------------------------- +if (NOT TARGET "@H5SZ3_PACKAGE@") + include (@PACKAGE_SHARE_INSTALL_DIR@/@H5SZ3_PACKAGE@@H5SZ3_PACKAGE_EXT@-targets.cmake) + set (H5SZ3_LIBRARIES "@H5SZ3_LIBRARIES_TO_EXPORT@") +endif () + +# Handle default component(static) : +if (NOT ${H5SZ3_PACKAGE_NAME}_FIND_COMPONENTS) + set (${H5SZ3_PACKAGE_NAME}_FIND_COMPONENTS static) + set (${H5SZ3_PACKAGE_NAME}_FIND_REQUIRED_static true) +endif () + +# Handle requested components: +list (REMOVE_DUPLICATES ${H5SZ3_PACKAGE_NAME}_FIND_COMPONENTS) +foreach (comp IN LISTS ${H5SZ3_PACKAGE_NAME}_FIND_COMPONENTS) + list (FIND ${H5SZ3_PACKAGE_NAME}_EXPORT_LIBRARIES "@H5SZ3_LIB_CORENAME@-${comp}" HAVE_COMP) + if (${HAVE_COMP} LESS 0) + set (${H5SZ3_PACKAGE_NAME}_${comp}_FOUND 0) + else () + set (${H5SZ3_PACKAGE_NAME}_${comp}_FOUND 1) + string(TOUPPER ${H5SZ3_PACKAGE_NAME}_${comp}_LIBRARY COMP_LIBRARY) + set (${COMP_LIBRARY} ${${COMP_LIBRARY}} @H5SZ3_LIB_CORENAME@-${comp}) + endif () +endforeach () + +check_required_components (${H5SZ3_PACKAGE_NAME}) diff --git a/SZ3/config/cmake/runTest.cmake b/SZ3/config/cmake/runTest.cmake new file mode 100644 index 00000000..5409772e --- /dev/null +++ b/SZ3/config/cmake/runTest.cmake @@ -0,0 +1,389 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# runTest.cmake executes a command and captures the output in a file. File is then compared +# against a reference file. Exit status of command can also be compared. +cmake_policy(SET CMP0007 NEW) + +# arguments checking +if (NOT TEST_PROGRAM) + message (FATAL_ERROR "Require TEST_PROGRAM to be defined") +endif () +if (NOT TEST_FOLDER) + message (FATAL_ERROR "Require TEST_FOLDER to be defined") +endif () +if (NOT TEST_OUTPUT) + message (FATAL_ERROR "Require TEST_OUTPUT to be defined") +endif () +if (NOT DEFINED TEST_EXPECT) + message (STATUS "Require TEST_EXPECT to be defined") +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}) +endif () + +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err) +endif () + +message (STATUS "COMMAND: ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS}") + +if (TEST_LIBRARY_DIRECTORY) + if (WIN32) + set (ENV{PATH} "$ENV{PATH};${TEST_LIBRARY_DIRECTORY}") + elseif (APPLE) + set (ENV{DYLD_LIBRARY_PATH} "$ENV{DYLD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + else () + set (ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") + endif () + set (ENV{HDF5_PLUGIN_PATH} "${TEST_LIBRARY_DIRECTORY}/plugins:${TEST_LIBRARY_DIRECTORY}/plugin:$ENV{HDF5_PLUGIN_PATH}") +endif () + +if (TEST_ENV_VAR) + set (ENV{${TEST_ENV_VAR}} "${TEST_ENV_VALUE}") + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (TRACE "ENV:${TEST_ENV_VAR}=$ENV{${TEST_ENV_VAR}}") + endif () +endif () + +if (NOT TEST_INPUT) + # run the test program, capture the stdout/stderr and the result var + execute_process ( + COMMAND ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS} + WORKING_DIRECTORY ${TEST_FOLDER} + RESULT_VARIABLE TEST_RESULT + OUTPUT_FILE ${TEST_OUTPUT} + ERROR_FILE ${TEST_OUTPUT}.err + OUTPUT_VARIABLE TEST_OUT + ) +else () + # run the test program with stdin, capture the stdout/stderr and the result var + execute_process ( + COMMAND ${TEST_EMULATOR} ${TEST_PROGRAM} ${TEST_ARGS} + WORKING_DIRECTORY ${TEST_FOLDER} + RESULT_VARIABLE TEST_RESULT + INPUT_FILE ${TEST_INPUT} + OUTPUT_FILE ${TEST_OUTPUT} + ERROR_FILE ${TEST_OUTPUT}.err + OUTPUT_VARIABLE TEST_OUT + ) +endif () + +if (TEST_REGEX) + # TEST_REGEX should always be matched + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + string (REGEX MATCH "${TEST_REGEX}" REGEX_MATCH ${TEST_STREAM}) + string (COMPARE EQUAL "${REGEX_MATCH}" "${TEST_MATCH}" REGEX_RESULT) + if (NOT REGEX_RESULT) + message (STATUS "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_MATCH}") + endif () +endif () + +message (STATUS "COMMAND Result: ${TEST_RESULT}") + +# if the .err file exists and ERRROR_APPEND is enabled +if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_ERROR) + list (LENGTH TEST_ERROR test_len) + if (test_len GREATER 0) + if (TEST_MASK_FILE) + STRING(REGEX REPLACE "CurrentDir is [^\n]+\n" "CurrentDir is (dir name)\n" TEST_ERROR "${TEST_ERROR}") + endif () + # remove special output + string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_ERROR "${TEST_ERROR}") + + if (NOT ERROR_APPEND) + # write back to original .err file + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_ERROR}") + else () + # append error output to the stdout output file + file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_ERROR}") + endif () + endif () +endif () + +# append the test result status with a predefined text +if (TEST_APPEND) + file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_APPEND} ${TEST_RESULT}\n") +endif () + +# if the return value is !=${TEST_EXPECT} bail out +if (NOT TEST_RESULT EQUAL TEST_EXPECT) + if (NOT TEST_NOERRDISPLAY) + if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + message (STATUS "Output :\n${TEST_STREAM}") + endif () + endif () + message (FATAL_ERROR "Failed: Test program ${TEST_PROGRAM} exited != ${TEST_EXPECT}.\n${TEST_ERROR}") +endif () + +message (STATUS "COMMAND Error: ${TEST_ERROR}") + +# remove special output +file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) +string (FIND TEST_STREAM "_pmi_alps" TEST_FIND_RESULT) +if (TEST_FIND_RESULT GREATER -1) + string (REGEX REPLACE "^.*_pmi_alps[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_STREAM}) +endif () + +# remove special error output +if (NOT TEST_ERRREF) + # the error stack has been appended to the output file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) +else () + # the error stack remains in the .err file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM) +endif () +string (FIND TEST_STREAM "no version information available" TEST_FIND_RESULT) +if (TEST_FIND_RESULT GREATER -1) + string (REGEX REPLACE "^.*no version information available[^\n]+\n" "" TEST_STREAM "${TEST_STREAM}") + # write back the changes to the original files + if (NOT TEST_ERRREF) + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") + else () + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}") + endif () +endif () + +# if the output file needs COMPRESSION text modified +if (TEST_MASK_MOD) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + string (REGEX REPLACE " SIZE [0-9]* .([0-9]*)[.][0-9]*:1 COMPRESSION." " SIZE XXX (\\1.XXXX:1 COMPRESSION)" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") +endif () + +# if the output file or the .err file needs to mask out error stack info +if (TEST_MASK_ERROR) + if (NOT TEST_ERRREF) + # the error stack has been appended to the output file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + else () + # the error stack remains in the .err file + file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM) + endif () + string (REGEX REPLACE "thread [0-9]*:" "thread (IDs):" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE ": ([^\n]*)[.]c " ": (file name) " TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE " line [0-9]*" " line (number)" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "v[1-9]*[.][0-9]*[.]" "version (number)." TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "[1-9]*[.][0-9]*[.][0-9]*[^)]*" "version (number)" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "H5Eget_auto[1-2]*" "H5Eget_auto(1 or 2)" TEST_STREAM "${TEST_STREAM}") + string (REGEX REPLACE "H5Eset_auto[1-2]*" "H5Eset_auto(1 or 2)" TEST_STREAM "${TEST_STREAM}") + # write back the changes to the original files + if (NOT TEST_ERRREF) + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") + else () + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}") + endif () +endif () + +# remove text from the output file +if (TEST_FILTER) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + string (REGEX REPLACE "${TEST_FILTER}" "${TEST_FILTER_REPLACE}" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") +endif () + +if (TEST_REF_FILTER) + #message (STATUS "TEST_REF_FILTER: ${TEST_APPEND}${TEST_REF_FILTER}") + file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + string (REGEX REPLACE "${TEST_REF_APPEND}" "${TEST_REF_FILTER}" TEST_STREAM "${TEST_STREAM}") + file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") +endif () + +# compare output files to references unless this must be skipped +set (TEST_COMPARE_RESULT 0) +if (NOT TEST_SKIP_COMPARE) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}") + file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + if (WIN32) + configure_file (${TEST_FOLDER}/${TEST_REFERENCE} ${TEST_FOLDER}/${TEST_REFERENCE}.tmp NEWLINE_STYLE CRLF) + if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}.tmp") + file(RENAME ${TEST_FOLDER}/${TEST_REFERENCE}.tmp ${TEST_FOLDER}/${TEST_REFERENCE}) + endif () + #file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) + #file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") + endif () + + if (NOT TEST_SORT_COMPARE) + # now compare the output with the reference + execute_process ( + COMMAND ${CMAKE_COMMAND} -E compare_files ${CMAKE_IGNORE_EOL} ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE} + RESULT_VARIABLE TEST_COMPARE_RESULT + ) + else () + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2) + list (SORT v1) + list (SORT v2) + if (NOT v1 STREQUAL v2) + set (TEST_COMPARE_RESULT 1) + endif () + endif () + + if (TEST_COMPARE_RESULT) + set (TEST_COMPARE_RESULT 0) + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act) + list (LENGTH test_act len_act) + file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref) + list (LENGTH test_ref len_ref) + if (NOT len_act EQUAL len_ref) + set (TEST_COMPARE_RESULT 1) + endif () + if (len_act GREATER 0 AND len_ref GREATER 0) + if (TEST_SORT_COMPARE) + list (SORT test_act) + list (SORT test_ref) + endif () + math (EXPR _FP_LEN "${len_ref} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + if (line GREATER_EQUAL len_act) + message (STATUS "COMPARE FAILED: ran out of lines in ${TEST_FOLDER}/${TEST_OUTPUT}") + set (TEST_COMPARE_RESULT 1) + break () + elseif (line GREATER_EQUAL len_ref) + message (STATUS "COMPARE FAILED: ran out of lines in ${TEST_FOLDER}/${TEST_REFERENCE}") + set (TEST_COMPARE_RESULT 1) + break () + else () + list (GET test_act ${line} str_act) + list (GET test_ref ${line} str_ref) + if (NOT str_act STREQUAL str_ref) + if (str_act) + set (TEST_COMPARE_RESULT 1) + message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") + endif () + endif () + endif () + endforeach () + else () + if (len_act EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty") + endif () + if (len_ref EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty") + endif () + endif () + endif () + endif () + + message (STATUS "COMPARE Result: ${TEST_COMPARE_RESULT}") + + # again, if return value is !=0 scream and shout + if (TEST_COMPARE_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}") + endif () + endif () + + # now compare the .err file with the error reference, if supplied + set (TEST_ERRREF_RESULT 0) + if (TEST_ERRREF) + file (READ ${TEST_FOLDER}/${TEST_ERRREF} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + if (WIN32) + configure_file (${TEST_FOLDER}/${TEST_ERRREF} ${TEST_FOLDER}/${TEST_ERRREF}.tmp NEWLINE_STYLE CRLF) + if (EXISTS "${TEST_FOLDER}/${TEST_ERRREF}.tmp") + file(RENAME ${TEST_FOLDER}/${TEST_ERRREF}.tmp ${TEST_FOLDER}/${TEST_ERRREF}) + endif () + #file (READ ${TEST_FOLDER}/${TEST_ERRREF} TEST_STREAM) + #file (WRITE ${TEST_FOLDER}/${TEST_ERRREF} "${TEST_STREAM}") + endif () + + # now compare the error output with the error reference + execute_process ( + COMMAND ${CMAKE_COMMAND} -E compare_files ${CMAKE_IGNORE_EOL} ${TEST_FOLDER}/${TEST_OUTPUT}.err ${TEST_FOLDER}/${TEST_ERRREF} + RESULT_VARIABLE TEST_ERRREF_RESULT + ) + if (TEST_ERRREF_RESULT) + set (TEST_ERRREF_RESULT 0) + file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT}.err test_act) + list (LENGTH test_act len_act) + file (STRINGS ${TEST_FOLDER}/${TEST_ERRREF} test_ref) + list (LENGTH test_ref len_ref) + math (EXPR _FP_LEN "${len_ref} - 1") + if (len_act GREATER 0 AND len_ref GREATER 0) + math (EXPR _FP_LEN "${len_ref} - 1") + foreach (line RANGE 0 ${_FP_LEN}) + list (GET test_act ${line} str_act) + list (GET test_ref ${line} str_ref) + if (NOT str_act STREQUAL str_ref) + if (str_act) + set (TEST_ERRREF_RESULT 1) + message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") + endif () + endif () + endforeach () + else () + if (len_act EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT}.err is empty") + endif () + if (len_ref EQUAL 0) + message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_ERRREF} is empty") + endif () + endif () + if (NOT len_act EQUAL len_ref) + set (TEST_ERRREF_RESULT 1) + endif () + endif () + endif () + + message (STATUS "COMPARE Result: ${TEST_ERRREF_RESULT}") + + # again, if return value is !=0 scream and shout + if (TEST_ERRREF_RESULT) + message (FATAL_ERROR "Failed: The error output of ${TEST_OUTPUT}.err did not match ${TEST_ERRREF}") + endif () + endif () +endif () + +set (TEST_GREP_RESULT 0) +if (TEST_GREP_COMPARE) + # now grep the output with the reference + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + list (LENGTH TEST_STREAM test_len) + if (test_len GREATER 0) + # TEST_REFERENCE should always be matched + string (REGEX MATCH "${TEST_REFERENCE}" TEST_MATCH ${TEST_STREAM}) + string (COMPARE EQUAL "${TEST_REFERENCE}" "${TEST_MATCH}" TEST_GREP_RESULT) + if (NOT TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}") + endif () + + if (TEST_FILTER) + string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) + if (TEST_EXPECT) + # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match + string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) + if (TEST_GREP_RESULT) + message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") + endif () + endif () + endif () + endif () +endif () + +# dump the output unless nodisplay option is set +if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + execute_process ( + COMMAND ${CMAKE_COMMAND} -E echo ${TEST_STREAM} + RESULT_VARIABLE TEST_RESULT + ) +endif () + +# everything went fine... +message (STATUS "${TEST_PROGRAM} Passed") + diff --git a/SZ3/config/toolchain/aarch64.cmake b/SZ3/config/toolchain/aarch64.cmake new file mode 100644 index 00000000..d061949e --- /dev/null +++ b/SZ3/config/toolchain/aarch64.cmake @@ -0,0 +1,18 @@ +set (TOOLCHAIN_PREFIX aarch64-linux-gnu) +set (CMAKE_SYSTEM_NAME Linux) +set (CMAKE_SYSTEM_PROCESSOR aarch64) +#set (CMAKE_ANDROID_ARCH_ABI x86_64) +#set (CMAKE_ANDROID_STANDALONE_TOOLCHAIN ${ANDROID_NDK}/build/cmake/android.toolchain.cmake) +set (CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set (CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +#set (CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) +set (CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) +set (CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) +set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set (CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set (CMAKE_CROSSCOMPILING_EMULATOR "qemu-aarch64-static;-L;/usr/aarch64-linux-gnu/" CACHE FILEPATH "Path to the emulator for the target system.") + +include_directories(/usr/${TOOLCHAIN_PREFIX}/include) + diff --git a/SZ3/config/toolchain/build32.cmake b/SZ3/config/toolchain/build32.cmake new file mode 100644 index 00000000..43ac5edb --- /dev/null +++ b/SZ3/config/toolchain/build32.cmake @@ -0,0 +1,90 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +if (WIN32) + set (CMAKE_SYSTEM_NAME Windows) + set (CMAKE_GENERATOR_PLATFORM "x86") +elseif(APPLE) + set (CMAKE_OSX_ARCHITECTURES "i386") +elseif(MINGW) + set (CMAKE_SYSTEM_NAME Windows) + set (TOOLCHAIN_PREFIX i686-w64-mingw32) + set (CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) + set (CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) + set (CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + set (CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags") + + set (LIB32 /usr/lib) # Fedora + + if (EXISTS "/usr/lib32") + set (LIB32 /usr/lib32) # Arch, Solus + endif () + + set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE) + set (CMAKE_LIBRARY_PATH ${LIB32} CACHE STRING "library search path" FORCE) + + # this is probably unlikely to be needed, but just in case + set (CMAKE_EXE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "executable linker flags" FORCE) + set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE) + set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags" FORCE) + + # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have + # to include the regular system .pc files as well (at the end), because some + # are not always present in the 32 bit directory + if (EXISTS "${LIB32}/pkgconfig") + set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig) + endif () + + set (CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + set (CMAKE_CROSSCOMPILING_EMULATOR wine) + + include_directories(/usr/${TOOLCHAIN_PREFIX}/include) + set (CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS On CACHE BOOL "Export windows symbols") +else () + set (CMAKE_SYSTEM_NAME Linux) + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags") + + set (LIB32 /usr/lib) # Fedora + + if (EXISTS "/usr/lib32") + set (LIB32 /usr/lib32) # Arch, Solus + endif () + + set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE) + set (CMAKE_LIBRARY_PATH ${LIB32} CACHE STRING "library search path" FORCE) + + # this is probably unlikely to be needed, but just in case + set (CMAKE_EXE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "executable linker flags" FORCE) + set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE) + set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags" FORCE) + + # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have + # to include the regular system .pc files as well (at the end), because some + # are not always present in the 32 bit directory + if (EXISTS "${LIB32}/pkgconfig") + set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig) + endif () +# where is the target environment + set (CMAKE_FIND_ROOT_PATH ${LIB32}) +# search for programs in the build host directories + set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +# for libraries and headers in the target directories + set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +endif () diff --git a/SZ3/config/toolchain/clang.cmake b/SZ3/config/toolchain/clang.cmake new file mode 100644 index 00000000..1242e134 --- /dev/null +++ b/SZ3/config/toolchain/clang.cmake @@ -0,0 +1,27 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Uncomment the following to use cross-compiling +#set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "clang") + +if(WIN32) + set (CMAKE_C_COMPILER clang-cl) + set (CMAKE_CXX_COMPILER clang-cl) +else() + set (CMAKE_C_COMPILER clang) + set (CMAKE_CXX_COMPILER clang++) +endif() +set (CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/toolchain/crayle.cmake b/SZ3/config/toolchain/crayle.cmake new file mode 100644 index 00000000..52248903 --- /dev/null +++ b/SZ3/config/toolchain/crayle.cmake @@ -0,0 +1,21 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# The following line will use cross-compiling +set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "CrayLinuxEnvironment") + +set (CMAKE_C_COMPILER cc) +set (CMAKE_Fortran_COMPILER ftn) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/toolchain/gcc.cmake b/SZ3/config/toolchain/gcc.cmake new file mode 100644 index 00000000..7f3c7586 --- /dev/null +++ b/SZ3/config/toolchain/gcc.cmake @@ -0,0 +1,22 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Uncomment the following line and the correct system name to use cross-compiling +#set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "GCC") + +set (CMAKE_C_COMPILER cc) +set (CMAKE_CXX_COMPILER c++) +set (CMAKE_Fortran_COMPILER gfortran) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/toolchain/icc.cmake b/SZ3/config/toolchain/icc.cmake new file mode 100644 index 00000000..97f6a649 --- /dev/null +++ b/SZ3/config/toolchain/icc.cmake @@ -0,0 +1,11 @@ +# Uncomment the following to use cross-compiling +#set(CMAKE_SYSTEM_NAME Linux) + +set(CMAKE_COMPILER_VENDOR "intel") + +set(CMAKE_C_COMPILER icc) +set(CMAKE_CXX_COMPILER icpc) +set(CMAKE_Fortran_COMPILER ifort) + +# the following is used if cross-compiling +set(CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/toolchain/intel.cmake b/SZ3/config/toolchain/intel.cmake new file mode 100644 index 00000000..f8f60b28 --- /dev/null +++ b/SZ3/config/toolchain/intel.cmake @@ -0,0 +1,11 @@ +# Uncomment the following to use cross-compiling +#set(CMAKE_SYSTEM_NAME Linux) + +set(CMAKE_COMPILER_VENDOR "intel") + +set(CMAKE_C_COMPILER icx) +set(CMAKE_CXX_COMPILER icpx) +set(CMAKE_Fortran_COMPILER ifx) + +# the following is used if cross-compiling +set(CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/config/toolchain/mingw64.cmake b/SZ3/config/toolchain/mingw64.cmake new file mode 100644 index 00000000..07517b44 --- /dev/null +++ b/SZ3/config/toolchain/mingw64.cmake @@ -0,0 +1,25 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +set (TOOLCHAIN_PREFIX x86_64-w64-mingw32) +set (CMAKE_SYSTEM_NAME Windows) +set (CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set (CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set (CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) +set (CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) +set (CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) +set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set (CMAKE_CROSSCOMPILING_EMULATOR wine64) + +include_directories(/usr/${TOOLCHAIN_PREFIX}/include) +set (CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS On CACHE BOOL "Export windows symbols") diff --git a/SZ3/config/toolchain/pgi.cmake b/SZ3/config/toolchain/pgi.cmake new file mode 100644 index 00000000..de67f59a --- /dev/null +++ b/SZ3/config/toolchain/pgi.cmake @@ -0,0 +1,22 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Uncomment the following to use cross-compiling +#set (CMAKE_SYSTEM_NAME Linux) + +set (CMAKE_COMPILER_VENDOR "PGI") + +set (CMAKE_C_COMPILER pgcc) +set (CMAKE_CXX_COMPILER pgc++) +set (CMAKE_Fortran_COMPILER pgf90) + +# the following is used if cross-compiling +set (CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/SZ3/example/CMakeLists.txt b/SZ3/example/CMakeLists.txt new file mode 100644 index 00000000..2b695de6 --- /dev/null +++ b/SZ3/example/CMakeLists.txt @@ -0,0 +1,179 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +cmake_minimum_required (VERSION 3.18) +project (HDF5SZ3_EX C) + +set (dyn_examples + h5ex_d_sz3 +) + +if (USE_SHARED_LIBS) + add_definitions (-DH5_BUILT_AS_DYNAMIC_LIB) +else () + add_definitions (-DH5_BUILT_AS_STATIC_LIB) +endif () + +#run-time loadable sz3 library examples +foreach (example ${dyn_examples}) + add_executable (${example} ${PROJECT_SOURCE_DIR}/${example}.c) + TARGET_C_PROPERTIES (${example} ${LIB_TYPE}) + target_link_libraries (${example} PRIVATE ${H5PL_HDF5_LINK_LIBS}) + if (NOT WIN32) + target_link_libraries (${example} PRIVATE ${CMAKE_DL_LIBS}) + endif () +endforeach () + +if (H5PL_BUILD_TESTING) + macro (ADD_H5_TEST testname) + add_test ( + NAME ${testname}-clearall + COMMAND ${CMAKE_COMMAND} + -E remove + ${testname}.out + ${testname}.out.err + ${testname}.ddl.out + ${testname}.ddl.out.err + ${testname}.h5 + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (${testname}-clearall PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "${testname}-clearall") + if (WIN32) + set (TESTLIBDIR "${HDF5_TOOLS_DIR}") + else () + set (TESTLIBDIR "${HDF5_LIBRARY_PATH}") + endif () + add_test ( + NAME ${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=${testname}.out" + -D "TEST_REFERENCE=${testname}.tst" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5SZ3_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (${testname} PROPERTIES DEPENDS ${testname}-clearall) + set (last_test "${testname}") + if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) + add_test ( + NAME H5DUMP-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=--enable-error-stack;-p;${testname}.h5" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${testname}.ddl.out" + -D "TEST_EXPECT=0" + -D "TEST_REFERENCE=${testname}.ddl" + -D "TEST_FILTER:STRING=PARAMS {[ -0-9]*}" + -D "TEST_FILTER_REPLACE:STRING=PARAMS { XXXX }" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5SZ3_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5DUMP-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS ${testname}) + set (last_test "H5DUMP-${testname}") + endif () + endmacro () + + macro (ADD_H5_UD_TEST testname resultcode resultfile) + # Remove any output file left over from previous test run + add_test ( + NAME H5SZ3_UD-${testname}-clearall-objects + COMMAND ${CMAKE_COMMAND} + -E remove + out-${testname}.${resultfile} + ${testname}.${resultfile}.out + ${testname}.${resultfile}.out.err + ${resultfile}-${testname}.out + ${resultfile}-${testname}.out.err + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5SZ3_UD-${testname}-clearall-objects PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "H5SZ3_UD-${testname}-clearall-objects") + if (WIN32) + set (${TESTLIBDIR} "${HDF5_TOOLS_DIR}") + else () + set (${TESTLIBDIR} "${HDF5_LIBRARY_PATH}") + endif () + if (HDF5_BUILD_TOOLS OR HDF5_PROVIDES_TOOLS) + add_test ( + NAME H5SZ3_UD-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_REPACK_EXECUTABLE}" + -D "TEST_ARGS:STRING=${ARGN};${PROJECT_BINARY_DIR}/${resultfile};${PROJECT_BINARY_DIR}/out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_FILTER:STRING=O?...ing file[^\n]+\n" + -D "TEST_OUTPUT=${testname}.${resultfile}.out" + -D "TEST_REFERENCE=${testname}.${resultfile}.tst" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5SZ3_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5SZ3_UD-${testname} PROPERTIES DEPENDS H5SZ3_UD-${testname}-clearall-objects) + add_test ( + NAME H5SZ3_UD-h5dump-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${H5PL_HDF5_DUMP_EXECUTABLE}" + -D "TEST_ARGS:STRING=-pH;out-${testname}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}-${testname}.ddl" + -D "TEST_FILTER:STRING=PARAMS {[ -0-9]*}" + -D "TEST_FILTER_REPLACE:STRING=PARAMS { XXXX }" + -D "TEST_LIBRARY_DIRECTORY=${TESTLIBDIR}" + -D "TEST_ENV_VAR=HDF5_PLUGIN_PATH" + -D "TEST_ENV_VALUE=${CMAKE_BINARY_DIR}/plugins" + -P "${H5SZ3_RESOURCES_DIR}/runTest.cmake" + ) + set_tests_properties (H5SZ3_UD-h5dump-${testname} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}" + DEPENDS H5SZ3_UD-${testname}) + set (last_test "H5SZ3_UD-h5dump-${testname}") + endif () + endmacro () + + # -------------------------------------------------------------------- + # Copy all the HDF5 files from the source directory into the test directory + # -------------------------------------------------------------------- + set (LIST_HDF5_TEST_FILES + h5repack_layout.h5 + ) + set (LIST_OTHER_TEST_FILES + h5ex_d_sz3.ddl + h5ex_d_sz3.tst + h5repack_layout.h5-ud_convert.ddl + ud_convert.h5repack_layout.h5.tst + ) + + foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) + HDFTEST_COPY_FILE("${PROJECT_SOURCE_DIR}/testfiles/${h5_file}" "${PROJECT_BINARY_DIR}/${h5_file}" "sz3_files") + endforeach () + add_custom_target(sz3_files ALL COMMENT "Copying files needed by sz3 tests" DEPENDS ${sz3_files_list}) + + ADD_H5_TEST (h5ex_d_sz3) + + #UD SZ3 with the default settings (ALGO_INTERP_LORENZO, absolute error bound 1e-3) + ADD_H5_UD_TEST (ud_convert 0 h5repack_layout.h5 --enable-error-stack -v -f UD=32024,0) +endif () diff --git a/SZ3/example/h5ex_d_sz3.c b/SZ3/example/h5ex_d_sz3.c new file mode 100644 index 00000000..8a40d3ee --- /dev/null +++ b/SZ3/example/h5ex_d_sz3.c @@ -0,0 +1,253 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of the HDF5 SZ3 filter plugin source. The full * + * copyright notice, including terms governing use, modification, and * + * terms governing use, modification, and redistribution, is contained in * + * the file COPYING, which can be found at the root of the SZ3 source code * + * distribution tree. If you do not have access to this file, you may * + * request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/************************************************************ + + This example shows how to write data and read it from a dataset + using SZ3 compression. + SZ3 filter is not available in HDF5. + The example uses a new feature available in HDF5 version 1.8.11 + to discover, load and register filters at run time. + + No cd_values are passed, so the filter's set_local callback + stores SZ3's default settings: the ALGO_INTERP_LORENZO + algorithm with an absolute error bound of 1e-3. SZ3 is lossy, + so the data read back is checked against that bound. + + ************************************************************/ + +#include "hdf5.h" +#include +#include + +#define FILENAME "h5ex_d_sz3.h5" +#define DATASET "DS1" +#define DIM0 32 +#define DIM1 64 +#define CHUNK0 4 +#define CHUNK1 8 +#define H5Z_FILTER_SZ3 32024 +#define SZ3_ABS_BOUND 1e-3 /* SZ3's default absolute error bound */ + +int +main(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + herr_t status; + htri_t avail; + H5Z_filter_t filter_id = 0; + char filter_name[256]; + hsize_t dims[2] = {DIM0, DIM1}, chunk[2] = {CHUNK0, CHUNK1}; + size_t nelmts = 10; /* number of elements in values_out */ + unsigned int flags; + unsigned filter_config; + unsigned int values_out[10] = {99, 99, 99, 99, 99, 99, 99, 99, 99, 99}; + float wdata[DIM0][DIM1]; /* Write buffer */ + float rdata[DIM0][DIM1]; /* Read buffer */ + float max; + double err, max_err; + hsize_t i, j; + int ret_value = 1; + + /* + * Initialize data. + */ + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) + wdata[i][j] = (float)(i * j) - (float)(j); + + /* + * Create a new file using the default properties. + */ + file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (file_id < 0) + goto done; + + /* + * Create dataspace. Setting maximum size to NULL sets the maximum + * size to be the current size. + */ + space_id = H5Screate_simple(2, dims, NULL); + if (space_id < 0) + goto done; + + /* + * Create the dataset creation property list, add the SZ3 + * compression filter and set the chunk size. + */ + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) + goto done; + + status = H5Pset_filter(dcpl_id, H5Z_FILTER_SZ3, H5Z_FLAG_MANDATORY, 0, NULL); + if (status < 0) + goto done; + + /* + * Check that filter is registered with the library now. + * If it is registered, retrieve filter's configuration. + */ + avail = H5Zfilter_avail(H5Z_FILTER_SZ3); + if (avail) { + status = H5Zget_filter_info(H5Z_FILTER_SZ3, &filter_config); + if ((filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED) && + (filter_config & H5Z_FILTER_CONFIG_DECODE_ENABLED)) + printf("sz3 filter is available for encoding and decoding.\n"); + } + else { + printf("H5Zfilter_avail - not found.\n"); + goto done; + } + status = H5Pset_chunk(dcpl_id, 2, chunk); + if (status < 0) + printf("failed to set chunk.\n"); + + /* + * Create the dataset. + */ + printf("....Create dataset ................\n"); + dset_id = H5Dcreate(file_id, DATASET, H5T_IEEE_F32LE, space_id, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dset_id < 0) { + printf("failed to create dataset.\n"); + goto done; + } + + /* + * Write the data to the dataset. + */ + printf("....Writing sz3 compressed data ................\n"); + status = H5Dwrite(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata[0]); + if (status < 0) + printf("failed to write data.\n"); + + /* + * Close and release resources. + */ + H5Dclose(dset_id); + dset_id = -1; + H5Pclose(dcpl_id); + dcpl_id = -1; + H5Sclose(space_id); + space_id = -1; + H5Fclose(file_id); + file_id = -1; + status = H5close(); + if (status < 0) { + printf("\nFAILED to close library\n"); + goto done; + } + + printf("....Close the file and reopen for reading ........\n"); + /* + * Now we begin the read section of this example. + */ + + /* + * Open file and dataset using the default properties. + */ + file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) + goto done; + + dset_id = H5Dopen(file_id, DATASET, H5P_DEFAULT); + if (dset_id < 0) + goto done; + + /* + * Retrieve dataset creation property list. + */ + dcpl_id = H5Dget_create_plist(dset_id); + if (dcpl_id < 0) + goto done; + + /* + * Retrieve and print the filter id, number of parameters and filter's name for SZ3. + * The first parameter is the SZ3 data format version, (major << 24) | (minor << 16) | (patch << 8); + * the others hold the SZ3 configuration the filter stored for the dataset. + */ + filter_id = H5Pget_filter2(dcpl_id, (unsigned)0, &flags, &nelmts, values_out, sizeof(filter_name), + filter_name, NULL); + printf("Filter info is available from the dataset creation property\n"); + printf(" Filter identifier is "); + switch (filter_id) { + case H5Z_FILTER_SZ3: + printf("%d\n", filter_id); + printf(" Number of parameters is %d with the value %u\n", (int)nelmts, values_out[0]); + printf(" To find more about the filter check %s\n", filter_name); + break; + default: + printf("Not expected filter\n"); + break; + } + + /* + * Read the data using the default properties. + */ + printf("....Reading sz3 compressed data ................\n"); + status = H5Dread(dset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata[0]); + if (status < 0) { + printf("failed to read data.\n"); + goto done; + } + + /* + * Find the maximum value in the dataset and the largest error, to verify + * that it was read correctly and within the error bound. + */ + max = rdata[0][0]; + max_err = 0.0; + for (i = 0; i < DIM0; i++) + for (j = 0; j < DIM1; j++) { + if (max < rdata[i][j]) + max = rdata[i][j]; + err = (double)rdata[i][j] - (double)wdata[i][j]; + if (err < 0) + err = -err; + if (err > max_err) + max_err = err; + } + /* + * Print the maximum value, rounded to the error bound. + */ + printf("Maximum value in %s is %6.2f\n", DATASET, max); + if (max_err > SZ3_ABS_BOUND) { + printf("Maximum absolute error %g exceeds the error bound %g\n", max_err, SZ3_ABS_BOUND); + goto done; + } + printf("Maximum absolute error is within the error bound %g\n", SZ3_ABS_BOUND); + /* + * Check that filter is registered with the library now. + */ + avail = H5Zfilter_avail(H5Z_FILTER_SZ3); + if (avail) + printf("sz3 filter is available now since H5Dread triggered loading of the filter.\n"); + + ret_value = 0; + +done: + /* + * Close and release resources. + */ + if (dcpl_id >= 0) + H5Pclose(dcpl_id); + if (dset_id >= 0) + H5Dclose(dset_id); + if (space_id >= 0) + H5Sclose(space_id); + if (file_id >= 0) + H5Fclose(file_id); + + return ret_value; +} diff --git a/SZ3/example/test/test.sh b/SZ3/example/test/test.sh new file mode 100644 index 00000000..aca64a98 --- /dev/null +++ b/SZ3/example/test/test.sh @@ -0,0 +1,258 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + +# This file is for use of h5cc created with the autotools process +# HDF5_HOME is expected to be set + +srcdir=.. +builddir=. +verbose=yes +nerrors=0 + +# HDF5 compile commands, assuming they are in your $PATH. +H5CC=$HDF5_HOME/bin/h5cc +LD_LIBRARY_PATH=$HDF5_HOME/lib +export LD_LIBRARY_PATH + +if ! test -f $H5CC; then + echo "Set paths for H5CC and LD_LIBRARY_PATH in test.sh" + echo "Set environment variable HDF5_HOME to the hdf5 install dir" + echo "h5cc was not found at $H5CC" + exit $EXIT_FAILURE +fi + +case $H5CC in +*/*) H5DUMP=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5dump/'`; + test -x $H5DUMP || H5DUMP=h5dump; + H5REPACK=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5repack/'`; + test -x $H5REPACK || H5REPACK=h5repack;; +*) H5DUMP=h5dump; + H5REPACK=h5repack;; +esac + +# Shell commands used in Makefiles +RM="rm -rf" +DIFF="diff -c" +CMP="cmp -s" +GREP='grep' +CP="cp -p" # Use -p to preserve mode,ownership,timestamps +DIRNAME='dirname' +LS='ls' +AWK='awk' + +# setup plugin path +ENVCMD="env HDF5_PLUGIN_PATH=$LD_LIBRARY_PATH/plugin" + +TESTDIR=$builddir + + +$H5CC -shlib $srcdir/h5ex_d_sz3.c -o h5ex_d_sz3 + +SRC_TESTFILES="$srcdir/testfiles" +LIST_TEST_FILES=" +$SRC_TESTFILES/h5repack_layout.h5 +$SRC_TESTFILES/h5ex_d_sz3.ddl +$SRC_TESTFILES/h5ex_d_sz3.tst +$SRC_TESTFILES/h5repack_layout.h5-ud_convert.ddl +$SRC_TESTFILES/ud_convert.h5repack_layout.h5.tst +" + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES="$LIST_TEST_FILES" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +# Print a $* message left justified in a field of 70 characters +# +MESSAGE() { + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + MESSAGE "Verifying $*" +} + +# This is different from $srcdir/../../bin/output_filter.sh +STDOUT_FILTER() { + result_file=$1 + tmp_file=/tmp/h5test_tmp_$$ + # Filter name of files. + cp $result_file $tmp_file + sed -e '/^Opening file/d' -e '/^Making file/d' \ + < $tmp_file > $result_file + # cleanup + rm -f $tmp_file +} + +# Compare the two text files +# PASS if same +# FAIL if different, and show the diff +# +# Assumed arguments: +# $1 is text file1 (expected output) +# $2 is text file2 (actual output) +CMP_OUTPUT() +{ + expect=$1 + actual=$2 + + VERIFY $@ + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " Expected Missing" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected output differs from actual output" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# ADD_H5_TEST +DUMPTEST() { + expect="$1" + actual="`basename $1 .ddl`.out" + actual_err="`basename $1 .ddl`.err" + shift + + # Run test. + TESTING $DUMPER $@ + ( + $ENVCMD $H5DUMP --enable-error-stack -p "$@" + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual + + # Clean up output file +# rm -f $actual $actual_err + +} + +REPACKTEST() +{ + infile=$1 + outfile=$2 + expect="$2.tst" + actual="$2.out1" + actual_err="$2.err1" + shift + shift + + # Run test. + TESTING H5REPACK $@ + ( + $ENVCMD $H5REPACK "$@" "out-$infile" "out-$outfile" + ) >$actual 2>$actual_err + RET=$? + STDOUT_FILTER $actual + cat $actual_err >> $actual + + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + + CMP_OUTPUT $expect $actual + fi +# rm -f $actual $actual_err +} + + +EXETEST() { + fname=$1 + expect="$2" + actual="`basename $2 .tst`.out1" + actual_err="`basename $2 .tst`.err1" + + # Run test. + TESTING $fname + ( + $ENVCMD `pwd`/$fname + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual +# rm -f $actual $actual_err +} + +################################ +######### THE TESTS ######### +################################ + +echo $ENVCMD + +COPY_TESTFILES_TO_TESTDIR + +EXETEST h5ex_d_sz3 h5ex_d_sz3.tst + +DUMPTEST h5ex_d_sz3.ddl h5ex_d_sz3.h5 + +REPACKTEST h5repack_layout.h5 out-ud_convert.h5repack_layout.h5 -v -f UD=32024,0 + +DUMPTEST h5repack_layout.h5-ud_convert.ddl -pH out-ud_convert.h5repack_layout.h5 + +#rm -f ud_convert.h5repack_layout.h5 + +echo "$nerrors tests failed in example" +exit $nerrors diff --git a/SZ3/example/test/testCM.sh b/SZ3/example/test/testCM.sh new file mode 100755 index 00000000..1e3a2551 --- /dev/null +++ b/SZ3/example/test/testCM.sh @@ -0,0 +1,258 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + +# This file is for use of h5cc created with the CMake process +# HDF5_HOME is expected to be set + +srcdir=.. +builddir=. +verbose=yes +nerrors=0 + +# HDF5 compile commands, assuming they are in your $PATH. +H5CC=$HDF5_HOME/bin/h5cc +LD_LIBRARY_PATH=$HDF5_HOME/lib +export LD_LIBRARY_PATH + +if ! test -f $H5CC; then + echo "Set paths for H5CC and LD_LIBRARY_PATH in test.sh" + echo "Set environment variable HDF5_HOME to the hdf5 install dir" + echo "h5cc was not found at $H5CC" + exit $EXIT_FAILURE +fi + +case $H5CC in +*/*) H5DUMP=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5dump-shared/'`; + test -x $H5DUMP || H5DUMP=h5dump-shared; + H5REPACK=`echo $H5CC | sed -e 's/\/[^/]*$/\/h5repack-shared/'`; + test -x $H5REPACK || H5REPACK=h5repack-shared;; +*) H5DUMP=h5dump-shared; + H5REPACK=h5repack-shared;; +esac + +# Shell commands used in Makefiles +RM="rm -rf" +DIFF="diff -c" +CMP="cmp -s" +GREP='grep' +CP="cp -p" # Use -p to preserve mode,ownership,timestamps +DIRNAME='dirname' +LS='ls' +AWK='awk' + +# setup plugin path +ENVCMD="env HDF5_PLUGIN_PATH=$LD_LIBRARY_PATH/plugin" + +TESTDIR=$builddir + + +$H5CC $srcdir/h5ex_d_sz3.c -o h5ex_d_sz3 + +SRC_TESTFILES="$srcdir/testfiles" +LIST_TEST_FILES=" +$SRC_TESTFILES/h5repack_layout.h5 +$SRC_TESTFILES/h5ex_d_sz3.ddl +$SRC_TESTFILES/h5ex_d_sz3.tst +$SRC_TESTFILES/h5repack_layout.h5-ud_convert.ddl +$SRC_TESTFILES/ud_convert.h5repack_layout.h5.tst +" + +# +# copy test files and expected output files from source dirs to test dir +# +COPY_TESTFILES="$LIST_TEST_FILES" + +COPY_TESTFILES_TO_TESTDIR() +{ + # copy test files. Used -f to make sure get a new copy + for tstfile in $COPY_TESTFILES + do + # ignore '#' comment + echo $tstfile | tr -d ' ' | grep '^#' > /dev/null + RET=$? + if [ $RET -eq 1 ]; then + # skip cp if srcdir is same as destdir + # this occurs when build/test performed in source dir and + # make cp fail + SDIR=`$DIRNAME $tstfile` + INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` + INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` + if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then + $CP -f $tstfile $TESTDIR + if [ $? -ne 0 ]; then + echo "Error: FAILED to copy $tstfile ." + + # Comment out this to CREATE expected file + exit $EXIT_FAILURE + fi + fi + fi + done +} + +# Print a $* message left justified in a field of 70 characters +# +MESSAGE() { + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY() { + MESSAGE "Verifying $*" +} + +# This is different from $srcdir/../../bin/output_filter.sh +STDOUT_FILTER() { + result_file=$1 + tmp_file=/tmp/h5test_tmp_$$ + # Filter name of files. + cp $result_file $tmp_file + sed -e '/^Opening file/d' -e '/^Making file/d' \ + < $tmp_file > $result_file + # cleanup + rm -f $tmp_file +} + +# Compare the two text files +# PASS if same +# FAIL if different, and show the diff +# +# Assumed arguments: +# $1 is text file1 (expected output) +# $2 is text file2 (actual output) +CMP_OUTPUT() +{ + expect=$1 + actual=$2 + + VERIFY $@ + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " Expected Missing" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected output differs from actual output" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# ADD_H5_TEST +DUMPTEST() { + expect="$1" + actual="`basename $1 .ddl`.out" + actual_err="`basename $1 .ddl`.err" + shift + + # Run test. + TESTING $DUMPER $@ + ( + $ENVCMD $H5DUMP --enable-error-stack -p "$@" + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual + + # Clean up output file +# rm -f $actual $actual_err + +} + +REPACKTEST() +{ + infile=$1 + outfile=$2 + expect="$2.tst" + actual="$2.out1" + actual_err="$2.err1" + shift + shift + + # Run test. + TESTING H5REPACK $@ + ( + $ENVCMD $H5REPACK "$@" "out-$infile" "out-$outfile" + ) >$actual 2>$actual_err + RET=$? + STDOUT_FILTER $actual + cat $actual_err >> $actual + + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + + CMP_OUTPUT $expect $actual + fi +# rm -f $actual $actual_err +} + + +EXETEST() { + fname=$1 + expect="$2" + actual="`basename $2 .tst`.out1" + actual_err="`basename $2 .tst`.err1" + + # Run test. + TESTING $fname + ( + $ENVCMD `pwd`/$fname + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + CMP_OUTPUT $expect $actual +# rm -f $actual $actual_err +} + +################################ +######### THE TESTS ######### +################################ + +echo $ENVCMD + +COPY_TESTFILES_TO_TESTDIR + +EXETEST h5ex_d_sz3 h5ex_d_sz3.tst + +DUMPTEST h5ex_d_sz3.ddl h5ex_d_sz3.h5 + +REPACKTEST h5repack_layout.h5 out-ud_convert.h5repack_layout.h5 -v -f UD=32024,0 + +DUMPTEST h5repack_layout.h5-ud_convert.ddl -pH out-ud_convert.h5repack_layout.h5 + +#rm -f ud_convert.h5repack_layout.h5 + +echo "$nerrors tests failed in example" +exit $nerrors diff --git a/SZ3/example/testfiles/h5ex_d_sz3.ddl b/SZ3/example/testfiles/h5ex_d_sz3.ddl new file mode 100644 index 00000000..364943d6 --- /dev/null +++ b/SZ3/example/testfiles/h5ex_d_sz3.ddl @@ -0,0 +1,209 @@ +HDF5 "h5ex_d_sz3.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 32, 64 ) / ( 32, 64 ) } + STORAGE_LAYOUT { + CHUNKED ( 4, 8 ) + SIZE 9015 (0.909:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + DATA { + (0,0): 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, + (0,15): -15, -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, + (0,27): -27, -28, -29, -30, -31, -32, -33, -34, -35, -36, -37, -38, + (0,39): -39, -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50, + (0,51): -51, -52, -53, -54, -55, -56, -57, -58, -59, -60, -61, -62, + (0,63): -63, + (1,0): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,21): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,42): 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + (1,63): 0, + (2,0): 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + (2,18): 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + (2,34): 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + (2,50): 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + (3,0): 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, + (3,17): 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, + (3,33): 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, + (3,49): 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, + (3,62): 124, 126, + (4,0): 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, + (4,17): 51, 54, 57, 60, 63, 66, 69, 72, 75, 78, 81, 84, 87, 90, 93, 96, + (4,33): 99, 102, 105, 108, 111, 114, 117, 120, 123, 126, 129, 132, 135, + (4,46): 138, 141, 144, 147, 150, 153, 156, 159, 162, 165, 168, 171, + (4,58): 174, 177, 180, 183, 186, 189, + (5,0): 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, + (5,17): 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, + (5,31): 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, + (5,43): 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, + (5,55): 220, 224, 228, 232, 236, 240, 244, 248, 252, + (6,0): 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, + (6,16): 80, 85, 90, 95, 100, 105, 110, 115, 120, 125, 130, 135, 140, + (6,29): 145, 150, 155, 160, 165, 170, 175, 180, 185, 190, 195, 200, + (6,41): 205, 210, 215, 220, 225, 230, 235, 240, 245, 250, 255, 260, + (6,53): 265, 270, 275, 280, 285, 290, 295, 300, 305, 310, 315, + (7,0): 0, 6, 12, 18, 24, 30, 36, 42, 48, 54, 60, 66, 72, 78, 84, 90, + (7,16): 96, 102, 108, 114, 120, 126, 132, 138, 144, 150, 156, 162, 168, + (7,29): 174, 180, 186, 192, 198, 204, 210, 216, 222, 228, 234, 240, + (7,41): 246, 252, 258, 264, 270, 276, 282, 288, 294, 300, 306, 312, + (7,53): 318, 324, 330, 336, 342, 348, 354, 360, 366, 372, 378, + (8,0): 0, 7, 14, 21, 28, 35, 42, 49, 56, 63, 70, 77, 84, 91, 98, 105, + (8,16): 112, 119, 126, 133, 140, 147, 154, 161, 168, 175, 182, 189, + (8,28): 196, 203, 210, 217, 224, 231, 238, 245, 252, 259, 266, 273, + (8,40): 280, 287, 294, 301, 308, 315, 322, 329, 336, 343, 350, 357, + (8,52): 364, 371, 378, 385, 392, 399, 406, 413, 420, 427, 434, 441, + (9,0): 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, + (9,16): 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, + (9,28): 224, 232, 240, 248, 256, 264, 272, 280, 288, 296, 304, 312, + (9,40): 320, 328, 336, 344, 352, 360, 368, 376, 384, 392, 400, 408, + (9,52): 416, 424, 432, 440, 448, 456, 464, 472, 480, 488, 496, 504, + (10,0): 0, 9, 18, 27, 36, 45, 54, 63, 72, 81, 90, 99, 108, 117, 126, + (10,15): 135, 144, 153, 162, 171, 180, 189, 198, 207, 216, 225, 234, + (10,27): 243, 252, 261, 270, 279, 288, 297, 306, 315, 324, 333, 342, + (10,39): 351, 360, 369, 378, 387, 396, 405, 414, 423, 432, 441, 450, + (10,51): 459, 468, 477, 486, 495, 504, 513, 522, 531, 540, 549, 558, + (10,63): 567, + (11,0): 0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, + (11,15): 150, 160, 170, 180, 190, 200, 210, 220, 230, 240, 250, 260, + (11,27): 270, 280, 290, 300, 310, 320, 330, 340, 350, 360, 370, 380, + (11,39): 390, 400, 410, 420, 430, 440, 450, 460, 470, 480, 490, 500, + (11,51): 510, 520, 530, 540, 550, 560, 570, 580, 590, 600, 610, 620, + (11,63): 630, + (12,0): 0, 11, 22, 33, 44, 55, 66, 77, 88, 99, 110, 121, 132, 143, 154, + (12,15): 165, 176, 187, 198, 209, 220, 231, 242, 253, 264, 275, 286, + (12,27): 297, 308, 319, 330, 341, 352, 363, 374, 385, 396, 407, 418, + (12,39): 429, 440, 451, 462, 473, 484, 495, 506, 517, 528, 539, 550, + (12,51): 561, 572, 583, 594, 605, 616, 627, 638, 649, 660, 671, 682, + (12,63): 693, + (13,0): 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, + (13,14): 168, 180, 192, 204, 216, 228, 240, 252, 264, 276, 288, 300, + (13,26): 312, 324, 336, 348, 360, 372, 384, 396, 408, 420, 432, 444, + (13,38): 456, 468, 480, 492, 504, 516, 528, 540, 552, 564, 576, 588, + (13,50): 600, 612, 624, 636, 648, 660, 672, 684, 696, 708, 720, 732, + (13,62): 744, 756, + (14,0): 0, 13, 26, 39, 52, 65, 78, 91, 104, 117, 130, 143, 156, 169, + (14,14): 182, 195, 208, 221, 234, 247, 260, 273, 286, 299, 312, 325, + (14,26): 338, 351, 364, 377, 390, 403, 416, 429, 442, 455, 468, 481, + (14,38): 494, 507, 520, 533, 546, 559, 572, 585, 598, 611, 624, 637, + (14,50): 650, 663, 676, 689, 702, 715, 728, 741, 754, 767, 780, 793, + (14,62): 806, 819, + (15,0): 0, 14, 28, 42, 56, 70, 84, 98, 112, 126, 140, 154, 168, 182, + (15,14): 196, 210, 224, 238, 252, 266, 280, 294, 308, 322, 336, 350, + (15,26): 364, 378, 392, 406, 420, 434, 448, 462, 476, 490, 504, 518, + (15,38): 532, 546, 560, 574, 588, 602, 616, 630, 644, 658, 672, 686, + (15,50): 700, 714, 728, 742, 756, 770, 784, 798, 812, 826, 840, 854, + (15,62): 868, 882, + (16,0): 0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, + (16,14): 210, 225, 240, 255, 270, 285, 300, 315, 330, 345, 360, 375, + (16,26): 390, 405, 420, 435, 450, 465, 480, 495, 510, 525, 540, 555, + (16,38): 570, 585, 600, 615, 630, 645, 660, 675, 690, 705, 720, 735, + (16,50): 750, 765, 780, 795, 810, 825, 840, 855, 870, 885, 900, 915, + (16,62): 930, 945, + (17,0): 0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, + (17,14): 224, 240, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, + (17,26): 416, 432, 448, 464, 480, 496, 512, 528, 544, 560, 576, 592, + (17,38): 608, 624, 640, 656, 672, 688, 704, 720, 736, 752, 768, 784, + (17,50): 800, 816, 832, 848, 864, 880, 896, 912, 928, 944, 960, 976, + (17,62): 992, 1008, + (18,0): 0, 17, 34, 51, 68, 85, 102, 119, 136, 153, 170, 187, 204, 221, + (18,14): 238, 255, 272, 289, 306, 323, 340, 357, 374, 391, 408, 425, + (18,26): 442, 459, 476, 493, 510, 527, 544, 561, 578, 595, 612, 629, + (18,38): 646, 663, 680, 697, 714, 731, 748, 765, 782, 799, 816, 833, + (18,50): 850, 867, 884, 901, 918, 935, 952, 969, 986, 1003, 1020, 1037, + (18,62): 1054, 1071, + (19,0): 0, 18, 36, 54, 72, 90, 108, 126, 144, 162, 180, 198, 216, 234, + (19,14): 252, 270, 288, 306, 324, 342, 360, 378, 396, 414, 432, 450, + (19,26): 468, 486, 504, 522, 540, 558, 576, 594, 612, 630, 648, 666, + (19,38): 684, 702, 720, 738, 756, 774, 792, 810, 828, 846, 864, 882, + (19,50): 900, 918, 936, 954, 972, 990, 1008, 1026, 1044, 1062, 1080, + (19,61): 1098, 1116, 1134, + (20,0): 0, 19, 38, 57, 76, 95, 114, 133, 152, 171, 190, 209, 228, 247, + (20,14): 266, 285, 304, 323, 342, 361, 380, 399, 418, 437, 456, 475, + (20,26): 494, 513, 532, 551, 570, 589, 608, 627, 646, 665, 684, 703, + (20,38): 722, 741, 760, 779, 798, 817, 836, 855, 874, 893, 912, 931, + (20,50): 950, 969, 988, 1007, 1026, 1045, 1064, 1083, 1102, 1121, 1140, + (20,61): 1159, 1178, 1197, + (21,0): 0, 20, 40, 60, 80, 100, 120, 140, 160, 180, 200, 220, 240, 260, + (21,14): 280, 300, 320, 340, 360, 380, 400, 420, 440, 460, 480, 500, + (21,26): 520, 540, 560, 580, 600, 620, 640, 660, 680, 700, 720, 740, + (21,38): 760, 780, 800, 820, 840, 860, 880, 900, 920, 940, 960, 980, + (21,50): 1000, 1020, 1040, 1060, 1080, 1100, 1120, 1140, 1160, 1180, + (21,60): 1200, 1220, 1240, 1260, + (22,0): 0, 21, 42, 63, 84, 105, 126, 147, 168, 189, 210, 231, 252, 273, + (22,14): 294, 315, 336, 357, 378, 399, 420, 441, 462, 483, 504, 525, + (22,26): 546, 567, 588, 609, 630, 651, 672, 693, 714, 735, 756, 777, + (22,38): 798, 819, 840, 861, 882, 903, 924, 945, 966, 987, 1008, 1029, + (22,50): 1050, 1071, 1092, 1113, 1134, 1155, 1176, 1197, 1218, 1239, + (22,60): 1260, 1281, 1302, 1323, + (23,0): 0, 22, 44, 66, 88, 110, 132, 154, 176, 198, 220, 242, 264, 286, + (23,14): 308, 330, 352, 374, 396, 418, 440, 462, 484, 506, 528, 550, + (23,26): 572, 594, 616, 638, 660, 682, 704, 726, 748, 770, 792, 814, + (23,38): 836, 858, 880, 902, 924, 946, 968, 990, 1012, 1034, 1056, + (23,49): 1078, 1100, 1122, 1144, 1166, 1188, 1210, 1232, 1254, 1276, + (23,59): 1298, 1320, 1342, 1364, 1386, + (24,0): 0, 23, 46, 69, 92, 115, 138, 161, 184, 207, 230, 253, 276, 299, + (24,14): 322, 345, 368, 391, 414, 437, 460, 483, 506, 529, 552, 575, + (24,26): 598, 621, 644, 667, 690, 713, 736, 759, 782, 805, 828, 851, + (24,38): 874, 897, 920, 943, 966, 989, 1012, 1035, 1058, 1081, 1104, + (24,49): 1127, 1150, 1173, 1196, 1219, 1242, 1265, 1288, 1311, 1334, + (24,59): 1357, 1380, 1403, 1426, 1449, + (25,0): 0, 24, 48, 72, 96, 120, 144, 168, 192, 216, 240, 264, 288, 312, + (25,14): 336, 360, 384, 408, 432, 456, 480, 504, 528, 552, 576, 600, + (25,26): 624, 648, 672, 696, 720, 744, 768, 792, 816, 840, 864, 888, + (25,38): 912, 936, 960, 984, 1008, 1032, 1056, 1080, 1104, 1128, 1152, + (25,49): 1176, 1200, 1224, 1248, 1272, 1296, 1320, 1344, 1368, 1392, + (25,59): 1416, 1440, 1464, 1488, 1512, + (26,0): 0, 25, 50, 75, 100, 125, 150, 175, 200, 225, 250, 275, 300, + (26,13): 325, 350, 375, 400, 425, 450, 475, 500, 525, 550, 575, 600, + (26,25): 625, 650, 675, 700, 725, 750, 775, 800, 825, 850, 875, 900, + (26,37): 925, 950, 975, 1000, 1025, 1050, 1075, 1100, 1125, 1150, 1175, + (26,48): 1200, 1225, 1250, 1275, 1300, 1325, 1350, 1375, 1400, 1425, + (26,58): 1450, 1475, 1500, 1525, 1550, 1575, + (27,0): 0, 26, 52, 78, 104, 130, 156, 182, 208, 234, 260, 286, 312, + (27,13): 338, 364, 390, 416, 442, 468, 494, 520, 546, 572, 598, 624, + (27,25): 650, 676, 702, 728, 754, 780, 806, 832, 858, 884, 910, 936, + (27,37): 962, 988, 1014, 1040, 1066, 1092, 1118, 1144, 1170, 1196, + (27,47): 1222, 1248, 1274, 1300, 1326, 1352, 1378, 1404, 1430, 1456, + (27,57): 1482, 1508, 1534, 1560, 1586, 1612, 1638, + (28,0): 0, 27, 54, 81, 108, 135, 162, 189, 216, 243, 270, 297, 324, + (28,13): 351, 378, 405, 432, 459, 486, 513, 540, 567, 594, 621, 648, + (28,25): 675, 702, 729, 756, 783, 810, 837, 864, 891, 918, 945, 972, + (28,37): 999, 1026, 1053, 1080, 1107, 1134, 1161, 1188, 1215, 1242, + (28,47): 1269, 1296, 1323, 1350, 1377, 1404, 1431, 1458, 1485, 1512, + (28,57): 1539, 1566, 1593, 1620, 1647, 1674, 1701, + (29,0): 0, 28, 56, 84, 112, 140, 168, 196, 224, 252, 280, 308, 336, + (29,13): 364, 392, 420, 448, 476, 504, 532, 560, 588, 616, 644, 672, + (29,25): 700, 728, 756, 784, 812, 840, 868, 896, 924, 952, 980, 1008, + (29,37): 1036, 1064, 1092, 1120, 1148, 1176, 1204, 1232, 1260, 1288, + (29,47): 1316, 1344, 1372, 1400, 1428, 1456, 1484, 1512, 1540, 1568, + (29,57): 1596, 1624, 1652, 1680, 1708, 1736, 1764, + (30,0): 0, 29, 58, 87, 116, 145, 174, 203, 232, 261, 290, 319, 348, + (30,13): 377, 406, 435, 464, 493, 522, 551, 580, 609, 638, 667, 696, + (30,25): 725, 754, 783, 812, 841, 870, 899, 928, 957, 986, 1015, 1044, + (30,37): 1073, 1102, 1131, 1160, 1189, 1218, 1247, 1276, 1305, 1334, + (30,47): 1363, 1392, 1421, 1450, 1479, 1508, 1537, 1566, 1595, 1624, + (30,57): 1653, 1682, 1711, 1740, 1769, 1798, 1827, + (31,0): 0, 30, 60, 90, 120, 150, 180, 210, 240, 270, 300, 330, 360, + (31,13): 390, 420, 450, 480, 510, 540, 570, 600, 630, 660, 690, 720, + (31,25): 750, 780, 810, 840, 870, 900, 930, 960, 990, 1020, 1050, 1080, + (31,37): 1110, 1140, 1170, 1200, 1230, 1260, 1290, 1320, 1350, 1380, + (31,47): 1410, 1440, 1470, 1500, 1530, 1560, 1590, 1620, 1650, 1680, + (31,57): 1710, 1740, 1770, 1800, 1830, 1860, 1890 + } + } +} +} diff --git a/SZ3/example/testfiles/h5ex_d_sz3.tst b/SZ3/example/testfiles/h5ex_d_sz3.tst new file mode 100644 index 00000000..d3622b4c --- /dev/null +++ b/SZ3/example/testfiles/h5ex_d_sz3.tst @@ -0,0 +1,12 @@ +sz3 filter is available for encoding and decoding. +....Create dataset ................ +....Writing sz3 compressed data ................ +....Close the file and reopen for reading ........ +Filter info is available from the dataset creation property + Filter identifier is 32024 + Number of parameters is 10 with the value 50593792 + To find more about the filter check H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 +....Reading sz3 compressed data ................ +Maximum value in DS1 is 1890.00 +Maximum absolute error is within the error bound 0.001 +sz3 filter is available now since H5Dread triggered loading of the filter. diff --git a/SZ3/example/testfiles/h5repack_layout.h5 b/SZ3/example/testfiles/h5repack_layout.h5 new file mode 100644 index 0000000000000000000000000000000000000000..2cc44b1405e955fde6483ccb2ddf2268b0f1d4de GIT binary patch literal 27280 zcmeI)1=Lm5+9=?U5=2o1R7_A5P*6z=>F)0C?(XjH?(XjH?(XjHhWl*!IQJZXk8;Po zXZ+*d%VIox)m(e8Z>~A#dfzqJyJLnl=^}*xIDAlG3KuRYbWp_K7hk?S51x8dda>o@ z`pf+#f)_#r&pQOqCxm>l5F#k>FDNK#@b>qD-~W&0^5jaJHb}{r@BeZhd|1$gP%oA; zyxj15{{K}CWJsGbN8k@G5d2raY;PNa+u6&?C{w#^o%mJ)^8~^3gu(Mf!Smv!E7Yx4 z*~&lGt5>a2vUHt)TCY~8V)?q&>(%wN1l@;QMiP$o>~!fA77 z%bq4gP}tzDJ|(z6{9fo!US9U%8!t}kPfEPl^s-+z2=i}?{Jtal%iV9hT=?_(%fGY3 z-<10~`0%JX|NMPkZp{$95IyY6E&qG-|NQkl5qydN`g;E7&!fZ_FKX()`#es3_rG!; z9fB`!h@clODa4Cz7QFU9`_IdE_VPaQ@A~0C`fI2cZ}g9T7cx|dUtfGy;Ccn_f4}#X zf9`h)lSIq*pTABKi@i9FzrIc{%KYE?8W_U@!(d>@3RH4ns0j=Yfi4}6Hwj#Tw+IY^ z5qO7p35;X!5sAnIF4p^ez=s6JuaEecPxzG2_?&1&C(seTBnDscH8F`rpdEd~x5ObX zfd>0MKM;?={UmTN3EW2#k(eL(iNHAa3xT$jgx^R?GLn;ml%ygxX%OfHr6WBV$VetK zlZC8gBRhecUSNpKMQ-wtmwe=>00k*TVTw?cK%**72})9m(v+brY(34*D zrVoATM}Gz|kUW_xyE&FaFbiy<_>qc$9*2~kVib`2~T;(a{>WO2m+&T zC_)p4Hwa5O-XuJ4@iq~7hj)p{dqg5KQFxyZ_>ic4#K(NXr+mieL?b$1@Fg+$im!=D zEMoHw-x7zoe8>0vKs@4;fP^F>F+cJXKl2N}l7!z#N-~m@f|R5pHG;UHw4@_F8OTT` zGLwa@xbXyKCkHvnMQ-wtmwe=>00k*TVTw?cViczYB`HN|%21Yal&1m}sYGR}P?c&_ zrv^2tMQ!R(mwMEv0S#$HW17&EW;CY-Eont-+R&DEw5J0d=|pF`(3Ngq#cl3zmwVjj0S|e^W1jGoXFMlx;}7IOAqhoj z!te%R3CEj+=PlkQ0`KrH5qXbDL?#OF^8p_cm5=zCPxzG2_?&1&=L^0h24C?tF^NTN zzTsQq5SQ=xo*#%ud=ik5L?q@%e&T0-;a8II8%aqWF`w) z$wqc^kds{GCJ%YZM}7)WkU|uu2t_GIaY|5Y(34*DrVoATM}Gz| zkUW_xyE&FaFbiy<_>qc$9*2~kVib`2~T;(a{@ts2tpEy(1hU)!V-=* z3C~-+O$6TIT_W-xk%&wb-sb~8Bq|^AF`w`$pYb`-2u#7lP~cQv=O5}IE$K*41~QU~ z%w!=e*~m@~a*~VODP6Jl zYEp~Z)S)i*s80hL(ul@1p()L1P77Mniq^EDE$wJe2RhP;&UB$G-RMpadeV#D^r0{P z=+6KKGKj$pVJO2G&Im>_iqVW=EaMo@1ST?x$xLA?)0oZ-W-^P}%waC`n9l+hvWUej zVJXX4&I(qtiq))PE$dj%1~#&Z&1_*S+t|(ycCw4z>|rna*v|nDa)`qm;V8#A&IwL( ziqo9oEay1S1uk-l%Ut0q*SO9NZgPv;+~F?wxX%L~@`%Sg;VI8}PN4ILAS9s(O&H!F zEa7;Q@Vv#_MBp9XB_i(;iO59ZeLmnrqVf?R^9i5w8J`o4=zPJK#NaEwCML0n%{P2Y z9OCjF-}3|Uh))6%l8D6o$WQ#tFZ@apej_Q#NKOh;l8V%%AuZ`hPX;oQiOggnE7{0S z4sw!<+~grI`N&TJ3Q~x|6rm`^C{77VQi{@)p)BPnPX#JciON)=D%Ge?4Qf)0+SH*g z^{7t+8q$cyG@&WYXif`S(u&r!p)KubPX{{EiOzJPE8XZ$4|>vz-t?g_{pimC1~Q1j z3}Gn47|sYrGK$fRVJzbq&jcniiOEc1D$|(G3}!Nm+00=s^O(;97P5%NEMY0jSk4Mo zvWnHLVJ+)e&jvQKiOph4=Y@4~fc0e9R|&%4d8|G@|nbUlN0__?no+A~xUfEpdp;cYMze z#3MclNJt_Q^CLg;Gr#aFN%)PVBqKQ~NJ%PElZLdUBRv_&NG39qg{)*FJ2}WnE^?EH zyyPQ41t>@%3R8rl6r(sLC`l}a>$Rs8+g{e$qIy0EbEM_x@xy)le3s}e^7PEw- zV?7(#$R;+kg{^F3J3H9PE_Snrz3gK@2RO(f4s(Q~9OF1AILRqabB42=<2)C*$R#dw zg{xfSIybnlYxw6A~RXYN;a~SgPi0d zH+jfQKJrt5f)t`KMJP%!ic^A;l%h0cC`&oYQ-O+9qB2#eN;RregPPQ$Hg%{=J?hhd zhBTrvO=wCpn$v=ow4ya_XiGcV(}9k3qBC9SN;kUGgP!!FH+|?!Kl(F(fed0WLm0|1 zhBJbZjAArn7|S@uGl7XrVlq>h$~2}kgPF`?HglNEJm#~2g)Cw*OIXS>ma~GDtYS55 zSj#%rvw@9lVl!LV$~LyMgPrVRH+$I2KK65fgB;>8M>xtcj&p*OoZ>WRILkTCbAgLo z;xbpb$~CTYgPYvqHg~woJ?`^>hdkmjPk72Ro)Z+}b^ei39i$>PX-G>t(vyLVWFj+J z$VxV{lY^Y(A~$)+OFr^bfPxgFFhwXzF^W@yl9Zw}WhhHI%2R=gRH8Cfs7f`eQ-hk+ zqBeD?OFin-fQB@pF->SnGn&(amb9WZZD>n7+S7rKbfPm|=t?)b(}SM$qBni$OF#NE zfPoBRFhdy1ForXNk&I$AV;IXg#xsG5Oky%qn94M!GlQATVm5P_%RJ_@fQ2k#F-us= zGM2M~m8@blYgo%V*0X_)Y+^H8*vdAxvxA-NVmEu(%RcsVfP)<3Fh@AbF^+SBlbqr- zXE@6_&U1l_T;eiUxXLxIbAy}Q;x>1<%RTP%fQLNdF;95PGoBL^@^$`^QXQlsHEBpo zI?|JYjASA+S;$H@vXg_HI4f|8V?G-W7DIm%Okid3R9 zRj5ies#AlS)S@Q6 z^rAO?=u1EPGk}2%VlYD($}omAf{~13G-DXcIL0%9iA-WLQ<%y$rZa|!^2*vmflbAW>!;xI=z z$}x^}f|H!$G-o)=InHx|i(KL|SGdYGu5*K%+~PKOxXV56^MHpu;xSKn$}^txI{)ze zLp6L!48G!PViJqke8acIAuiwXJwFhS_#_}9iAc>6Q1&n=LCg# zoqwcM2dPL+8q$)E^kg6-naE5QvXYJLHNAm8eV=s#1;W)SxD{s7)Q}QjhvHpdpQDOcR>YjOMhUC9P;p8`{#2_H>{l zo#;##y3&pA^q?ob=uIE`(vSWOU?77S%n*h$jNy!6B%>J37{)S=@l0SMlbFmDrZSD` z%wQ(7n9UsKGLQKzU?GcG%o3KejODCgC97D?8rHIo^=x1xo7l`2wz7@w>|iIm*v%gH zvXA{7;2?)M%n^=qjN_c(B&Rsd8P0N!^IYH}m$=Lou5yj*+~6j+xXm5za*z8w;31EA z%oCpSjOPS}e4T%!R0pX@O&ZdYj`U<8Bbmrd7P69!?BpOPxyVf(@{*7I6rdo5C`=KG zQjFr1pd_UzO&Q8kj`CEXB9*926{=E=>eQenwWv)U>QayTG@v1kXiO8D(v0S`pe3zn zO&i+Mj`nn*Bc13>7rN4o?)0E1z35FJ`qGd73}7IG7|alcGK}GjU?ig$%^1cqj`2)j zB9oZR6s9tb>C9jzvzW~s<}#1@EMOsvSj-ZZvW(@dU?r)hZbx46w6 z?sAX&Jm4XZc+3-?@{H&Fcjq60?{$U<3jJTc*BL!jAUOH=(||ND_WiM3_&*%7ApO6V zi}Hv4FINv_`tx#se7^@-Umo#d{__1^mJ9u_yI-%pu7TGz@VW+G*TCxR!ipgMzdha{Ea7;QKz#N-AMhbj`G}ACgira5*nGpc#33%<@jX8fkNEsX zQj(FJ6r>~-sR@LoX~{uOa*>-n zNkn2&kdjoSCJhpq|MJ{0k8OF$M}7)WkU|uu2t}ztMJiF5DpaK!)u};En$VPHG^YhE zX+>+=(3U_x@)mCsfp>V9h`dK6BJ(-Xh|U*$NesT?Yhn_M1SBL8iTRPA_?ch$l_aDi zJsHSICNh(StYjlQg(yrBic*Z?l%OP~C`~o0Q-hk+qBeD?OFin-iq^EDE$wJe2RhP; z&h(={0~p941~Y`A3}ZNxn9LNWGL7lXU?#Je%`%p=f|aadHEUSQI@Ys`-Rxm6``FI` z4swXYoZ~zfxX2|gbA_v1<2sLc%oCpSjOPS}_}Az8?^u7Ah`dK6A`^x8`G61kk{Eo& z*Tf_ivH6B?iNlZl#LxV~uO#6&l9G(%WFj+J$VxV{lY^Y(A~(e-P6>(8$u4%YhrOKOB&Rsd8P0N!^IYH}ceu+v?(=|$ zJmN7=cuKgCulE1gKUMBCJ|`N{`GPNr!B>1uJmQmpgd`#{Kk^el^9yN6OFGh%fsAA# zGg-(=0SZ!x!W5w>#VAe*N>YWYRHHgIs7WnqQ-`{=pe3znO&i+Mj`nn*BYo&gKl(F( zfed0WLm0{gCh~XBA(tq#lw~Yu1uI#_YSyrpSAA}HM%lCc-Sfa#ea`o)-=7?$%?xG; zLm9?!Mlh05jAk0snZZnEF`GHeWghcc#cI~DmUXOW0~^`IX7;h4103WKhdIJgj&Ynz zT;>W_xyE&FaFbiy<{8fkQimZ3Nhm@ShBpXHBq9@q_xXSiiONTO%qPSo7P0w;Z;3-( zzT-n*a)9`||G?-#%7_l95o?$d=p zfMxo}???szAl1M7y{W(Z-M4@KeW}#tqGbyg)ZzC}2o?PMZ^2FZFLR#o&gU%x2mV#; z7sdYf`G60J%13<6Cw$6hd`>i?^95fLgFnWIv2FW?Z;3-(zTi8*s9my=xNCJ%YZM}7)WkU|uu2t_GIaY|5AZhTiM2TcCeFIjm3{Bdz51w=L9D?#c9rPmUEov0vEZ&Wv+0QYh33BH@U@a?r@iT z+~)xgdBkI$@RVmfCn&_f#`dADhv5yv5{@?s&s)4r1m59YBJv)Qh)fjT=L0?@Dj)GN zpYSQ4@j20m&KG=148G!PViJqke8acIAuiwXJwFhS_#_}9iAc> z6Q1&n=LCiP>liki{cjSUw|JWfyu-Uh1uOk(kO zF>yR);*)@cBqA|C@)JMv3$GeGr%^U7=}1ooGLnhRWFafr$W9J&l8fBr;h$sd0@e#s zh{6=1D8(pF2})9m(v+br{luNvF;QMNDr=+6KKGKj$pVJNSf158kMB9oZR6s9tb>C9jzvzW~s<}!~z%N3T` zzLaGwX9X)+#cI~DmUXOW0~^`IpXC%gY~RT)cC&}Q>|;L%ILNE!9%qz2%Q?<-fs0(? zGFQ0DHLi1mo800yceu+v?(=|$JmN7=c*--L6A1c4@T$4YtLFNzn)3ugkiZl?6a`Kv zC^TVsgRq3-O~UgQZxexcc$bL0Mh4=Y@*Ez`R9AtoYFpxnEW(Y$W#&AY3l2MH2 zbq?}62l?876q8uQ<{Q2x4srR8@A-jv#3um>Nkn3P+=(3WeG#AU8h4=Y@4~fc0e9R|&%4dAe>m1~D4l+VJ W7|AF`GlsE@V?3{OkpH)HkpBU1R2eV; literal 0 HcmV?d00001 diff --git a/SZ3/example/testfiles/h5repack_layout.h5-ud_convert.ddl b/SZ3/example/testfiles/h5repack_layout.h5-ud_convert.ddl new file mode 100644 index 00000000..2fced728 --- /dev/null +++ b/SZ3/example/testfiles/h5repack_layout.h5-ud_convert.ddl @@ -0,0 +1,158 @@ +HDF5 "out-ud_convert.h5repack_layout.h5" { +GROUP "/" { + DATASET "dset1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset2" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset3" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset4" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_chunk" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, H5S_UNLIMITED ) } + STORAGE_LAYOUT { + CHUNKED ( 20, 10 ) + SIZE 552 (5.797:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_compact" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_contiguous" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 40, 20 ) / ( 40, 20 ) } + STORAGE_LAYOUT { + CHUNKED ( 40, 20 ) + SIZE 135 (23.704:1 COMPRESSION) + } + FILTERS { + USER_DEFINED_FILTER { + FILTER_ID 32024 + COMMENT H5Z-SZ3-3.4.0 (data format 3.4.0); see https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + PARAMS { XXXX } + } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/SZ3/example/testfiles/ud_convert.h5repack_layout.h5.tst b/SZ3/example/testfiles/ud_convert.h5repack_layout.h5.tst new file mode 100644 index 00000000..64e56c1d --- /dev/null +++ b/SZ3/example/testfiles/ud_convert.h5repack_layout.h5.tst @@ -0,0 +1,15 @@ +No all objects to modify layout +All objects to apply filter are... + User Defined 32024 +Making new file ... +----------------------------------------- + Type Filter (Compression) Name +----------------------------------------- + group / + dset UD (23.704:1) /dset1 + dset UD (23.704:1) /dset2 + dset UD (23.704:1) /dset3 + dset UD (23.704:1) /dset4 + dset UD (5.797:1) /dset_chunk + dset UD (23.704:1) /dset_compact + dset UD (23.704:1) /dset_contiguous diff --git a/docs/PluginLibraries.txt b/docs/PluginLibraries.txt index 80db0c96..68a757b4 100644 --- a/docs/PluginLibraries.txt +++ b/docs/PluginLibraries.txt @@ -389,4 +389,39 @@ This has 1 optional value; Default h5repack parameter: UD=32015,0,0 ................................. +SZ3 Filter +================================= +Filter ID: 32024 + +Filter Description: + SZ3 is a modular error-bounded lossy compression framework for scientific + datasets. It compresses float, double and 1, 2, 4 or 8-byte integer + datasets in host byte order, with chunks of up to 4 dimensions longer + than 1, within a user-set error bound (absolute, relative, PSNR or L2 norm). + + The filter source is H5Z-SZ3, maintained in the SZ3 repository; the plugin + is built by SZ3's own CMake from the SZ3 source in SZ3/SZ3 (git subtree), + with the Zstd library bundled in SZ3. + +Link to the filter: + https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3 + +Link to the compression library: + https://github.com/szcompressor/SZ3 + +-------------------------------------------- +Compression Library Included: SZ3 version 3.4.0 + +Plugin cd_value option +-------------------------------------------- +cd_values[0] is the SZ3 data format version, (major << 24) | (minor << 16) | +(patch << 8), followed by the SZ3 configuration serialized by SZ3. Without +cd_values the filter uses SZ3's default settings: the ALGO_INTERP_LORENZO +algorithm with an absolute error bound of 1e-3. Use H5Pset_sz3() from the +H5Z-SZ3 library, or cdvalueHelper from SZ3, to produce cd_values for other +settings. Decompression reads the settings from the compressed data. + +Default h5repack parameter: UD=32024,0 +................................. + diff --git a/docs/RegisteredFilterPlugins.md b/docs/RegisteredFilterPlugins.md index 6b76c66b..50b82558 100644 --- a/docs/RegisteredFilterPlugins.md +++ b/docs/RegisteredFilterPlugins.md @@ -741,7 +741,7 @@ SZ3 is a modular error-bounded lossy compression framework for scientific datase This filter is documented, implemented, and maintained at: https://github.com/szcompressor/SZ3/tree/master/tools/H5Z-SZ3. -License: https://github.com/szcompressor/SZ/blob/master/copyright-and-BSD-license.txt +License: https://github.com/szcompressor/SZ3/blob/master/copyright-and-BSD-license.txt ##### Contact From b2303330cdfec355ca8819cde3ad0136add7ad49 Mon Sep 17 00:00:00 2001 From: Kai Zhao Date: Wed, 7 Oct 2026 02:33:30 -0400 Subject: [PATCH 3/3] Squashed 'SZ3/SZ3/' changes from 6141ed78..592df793 592df793 CMake: find the system Zstd with find_library; a static SZ3c links on Windows (#184) git-subtree-dir: SZ3/SZ3 git-subtree-split: 592df793eb596d952242aa9fec9cbe5e68782aae --- .github/workflows/packaging.yml | 24 ++++++++++++++++++++++++ CHANGELOG.md | 3 ++- CMakeLists.txt | 12 ++++++------ docs/advanced-guide.md | 4 +++- tools/sz3c/CMakeLists.txt | 4 ++++ tools/sz3c/include/sz3c.h | 4 +++- 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 08a53871..0fd8e053 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -79,6 +79,21 @@ jobs: shell: bash run: tools/H5Z-SZ3/test/filterSuite.sh "$PWD/_prefix" + # A parent that finds Zstd itself, as hdf5_plugins' ZSTD filter does, reads ZSTD_* from its cache. libzstd.pc is + # installed on this runner, so a pkg-config search for Zstd in SZ3 would leave them there. + - name: SZ3 as a subproject leaves the parent's ZSTD_* alone + shell: bash + run: | + mkdir -p _parent + printf 'cmake_minimum_required(VERSION 3.19)\nproject(parent CXX)\nadd_subdirectory(%s sz3)\n' "$PWD" > _parent/CMakeLists.txt + for bundled in ON OFF; do + rm -rf _parent/b + cmake -S _parent -B _parent/b -DSZ3_USE_BUNDLED_ZSTD=$bundled > /dev/null + if grep -E '^ZSTD_' _parent/b/CMakeCache.txt; then + echo "with SZ3_USE_BUNDLED_ZSTD=$bundled, SZ3 left the cache entries above in its parent"; exit 1 + fi + done + # BUILD_H5Z_FILTER is off by default, so a packaged SZ3 is likely to come without the filter; # find_package(SZ3 COMPONENTS hdf5sz3) must then decline and leave no targets behind. - name: An SZ3 installed without the filter @@ -308,6 +323,15 @@ jobs: export PATH="/c/vcpkg/installed/x64-windows/bin:$PATH" tools/test/consumeInstalledSZ3.sh "$PWD/_prefix" /c/vcpkg/installed/x64-windows + # What vcpkg's and Conan's static Windows packages are: a consumer of the static SZ3c must not see dllimport. + - name: A static SZ3 installed and consumed + shell: bash + run: | + cmake -S . -B build-static -DBUILD_SHARED_LIBS=OFF + cmake --build build-static --config ${{ env.BUILD_TYPE }} --parallel 4 + cmake --install build-static --config ${{ env.BUILD_TYPE }} --prefix "$PWD/_prefix_static" + tools/test/consumeInstalledSZ3.sh "$PWD/_prefix_static" + windows-mingw: name: Install and consume (Windows MinGW) runs-on: windows-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index ba19dafc..fd9dd28d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -265,7 +265,7 @@ Tag `v3.4.0`. The GitHub release lists #127–#183. - A Nix expression for NixOS. [#129](https://github.com/szcompressor/SZ3/pull/129) - `ALGO_BIOMD` is a new codec for molecular-dynamics coordinates `{frames, atoms, 3}` with an absolute bound: rigid water and bonded atoms are predicted from their geometry on an integer lattice. On 16 GROMACS trajectories at 5e-4 nm its compression ratio is 1.23 times that of the previous `ALGO_BIOMD` with one frame per chunk and 1.29 times with 20. Other shapes throw `std::invalid_argument` (in HDF5, creating the dataset fails); coordinates beyond its lattice (2048 nm at 5e-4 nm and 128 nm at 5e-5 nm for float) and NaN or Inf outside trailing fill frames go to lossless compression. [#169](https://github.com/szcompressor/SZ3/pull/169) - [docs/molecular-dynamics.md](docs/molecular-dynamics.md) for molecular-dynamics data, and the [advanced guide](docs/advanced-guide.md) for algorithms, building, CMake and packaging, the C++ API, OpenMP, floating point and contributing; the HDF5 filter's README covers using it in an application; the README is a quick start. [#173](https://github.com/szcompressor/SZ3/pull/173) -- `find_package(SZ3 COMPONENTS SZ3c)` provides `SZ3::SZ3c`, the SZ2-compatible C library. [#177](https://github.com/szcompressor/SZ3/pull/177) +- `find_package(SZ3 COMPONENTS SZ3c)` provides `SZ3::SZ3c`, the SZ2-compatible C library. A static one defines `SZ3C_STATIC` for its consumers, so it links on Windows. [#177](https://github.com/szcompressor/SZ3/pull/177), [#184](https://github.com/szcompressor/SZ3/pull/184) - Homebrew: `brew install szcompressor/tap/sz3`. [#176](https://github.com/szcompressor/SZ3/pull/176) - pysz builds against an installed SZ3 and its Zstd with `PYSZ_SZ3_PREFIX`. [#177](https://github.com/szcompressor/SZ3/pull/177) @@ -279,6 +279,7 @@ Tag `v3.4.0`. The GitHub release lists #127–#183. - `Config::load` and the lossless stages take the length of the buffer they read. [#145](https://github.com/szcompressor/SZ3/pull/145) - Integer data compressed by earlier versions decompresses only with those versions. 8-byte integers beyond ±2^53 are refused. [#165](https://github.com/szcompressor/SZ3/pull/165) - `cmake --install` no longer installs `sz3_customized_demo` or `share/SZ3/testfloat_8_8_128.dat`. +- A system Zstd is found with `find_library` (`SZ3_ZSTD_LIBRARY`), as an installed `SZ3Config.cmake` finds it, not with pkg-config, so SZ3 added as a subproject leaves the parent's `ZSTD_*` cache entries alone. [#184](https://github.com/szcompressor/SZ3/pull/184) - `libhdf5sz3` and `libSZ3c` export only their public functions, and carry an SOVERSION. [#158](https://github.com/szcompressor/SZ3/pull/158), [#159](https://github.com/szcompressor/SZ3/pull/159) - A static `libhdf5sz3` keeps `H5Pset_sz3`, `set_SZ3_conf_to_H5` and `get_SZ3_conf_from_H5` hidden in the shared library it is linked into, so they cannot take over another SZ3 copy's calls. `H5PLget_plugin_type` and `H5PLget_plugin_info` stay exported, as HDF5's `H5PLextern.h` declares them. [#174](https://github.com/szcompressor/SZ3/pull/174) - 32-bit builds (wasm32) and 64-bit builds decompress each other's data: every size in the stream is 8 bytes, as 64-bit builds already wrote it. [#177](https://github.com/szcompressor/SZ3/pull/177) diff --git a/CMakeLists.txt b/CMakeLists.txt index ea4136e6..6666e66e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,8 +32,6 @@ else () #-Wunused-parameter set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wreorder -Wsign-compare -Wsuggest-override -Wold-style-cast -Wcast-qual -Wextra-semi -Wnon-virtual-dtor -Wcast-align -Wunused-variable") endif () -find_package(PkgConfig) - configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/SZ3/version.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/SZ3/version.hpp) @@ -72,11 +70,13 @@ endif () # "${MSVC}" reads as ON under MSVC and OFF everywhere else. Keep the quotes. option(SZ3_USE_BUNDLED_ZSTD "prefer the bundled version of Zstd" "${MSVC}") -pkg_search_module(ZSTD libzstd) -if (ZSTD_FOUND AND NOT SZ3_USE_BUNDLED_ZSTD) - # Not PkgConfig::ZSTD: that imported target does not exist in the consumer's project. +if (NOT SZ3_USE_BUNDLED_ZSTD) + # The search SZ3Config.cmake repeats in the consumer's project. + find_library(SZ3_ZSTD_LIBRARY NAMES zstd) +endif () +if (SZ3_ZSTD_LIBRARY AND NOT SZ3_USE_BUNDLED_ZSTD) # BUILD_INTERFACE, and no include directory: both are paths on this machine only. - target_link_libraries(SZ3core INTERFACE $) + target_link_libraries(SZ3core INTERFACE $) else () # What this build uses, for SZ3Config.cmake: the option only states a preference. set(SZ3_USE_BUNDLED_ZSTD ON) diff --git a/docs/advanced-guide.md b/docs/advanced-guide.md index 4459fbc1..9228c69c 100644 --- a/docs/advanced-guide.md +++ b/docs/advanced-guide.md @@ -49,7 +49,8 @@ buffers with huge pages, which makes compressing and decompressing large fields ### Dependencies -- **Zstd.** pkg-config first, else the vendored Zstd in `tools/zstd`, built as the private static target `sz3_zstd` +- **Zstd.** `find_library(SZ3_ZSTD_LIBRARY NAMES zstd)` first, the search `SZ3Config.cmake` repeats for consumers, else + the vendored Zstd in `tools/zstd`, built as the private static target `sz3_zstd` with hidden symbols. Nothing is downloaded at configure time, so SZ3 builds offline. No SZ3 header includes `zstd.h` (`Lossless_zstd.hpp` declares the four functions it calls), so consumers need no Zstd include directory; keep it that way. @@ -77,6 +78,7 @@ buffers with huge pages, which makes compressing and decompressing large fields - A static `hdf5sz3` is position independent and defines `HDF5SZ3_STATIC` for its consumers. Its own functions are hidden in the shared library that links it; HDF5's `H5PLextern.h` makes `H5PLget_plugin_type` and `H5PLget_plugin_info` exported. +- A static `SZ3c` defines `SZ3C_STATIC` for its consumers, so `sz3c.h` does not mark its functions dllimport. - Install rules use `$`. `bin/sz3_smoke_test` exits 0 on a working install; Spack runs it. ## Using SZ3 from CMake diff --git a/tools/sz3c/CMakeLists.txt b/tools/sz3c/CMakeLists.txt index ca419d8a..c209a632 100644 --- a/tools/sz3c/CMakeLists.txt +++ b/tools/sz3c/CMakeLists.txt @@ -9,6 +9,10 @@ target_link_libraries( ) target_compile_definitions(SZ3c PRIVATE SZ3C_EXPORTS) +if (NOT BUILD_SHARED_LIBS) + # Otherwise sz3c.h marks the functions of a static SZ3c dllimport on Windows. + target_compile_definitions(SZ3c PUBLIC SZ3C_STATIC) +endif () set_target_properties(SZ3c PROPERTIES CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON) set_target_properties(SZ3c PROPERTIES VERSION ${SZ3_VERSION} SOVERSION ${SZ3_VERSION_MAJOR}.${SZ3_VERSION_MINOR}) diff --git a/tools/sz3c/include/sz3c.h b/tools/sz3c/include/sz3c.h index bf8e34a7..80d8adf9 100644 --- a/tools/sz3c/include/sz3c.h +++ b/tools/sz3c/include/sz3c.h @@ -36,7 +36,9 @@ /** End dataType in SZ2 (defines.h) **/ #ifdef _WIN32 -#ifdef SZ3C_EXPORTS +#if defined(SZ3C_STATIC) +#define SZ3C_API +#elif defined(SZ3C_EXPORTS) #define SZ3C_API __declspec(dllexport) #else #define SZ3C_API __declspec(dllimport)