From 2d0da432a17f441a0bd7ff69ef20d6e4e7df32c3 Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Mon, 5 Oct 2026 15:05:11 +0800 Subject: [PATCH 1/2] Propagate hash context allocation failures --- librhash/rhash.c | 1 + tests/test_init_alloc_failure.py | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/test_init_alloc_failure.py diff --git a/librhash/rhash.c b/librhash/rhash.c index 56c4aee5..612c524f 100644 --- a/librhash/rhash.c +++ b/librhash/rhash.c @@ -158,6 +158,7 @@ static rhash_context_ext* rhash_alloc_multi(size_t count, const unsigned hash_id RHASH_API rhash rhash_init_multi(size_t count, const unsigned hash_ids[]) { rhash_context_ext* ectx = rhash_alloc_multi(count, hash_ids, 1); + if (!ectx) return NULL; return &ectx->rc; /* return initialized rhash context */ } diff --git a/tests/test_init_alloc_failure.py b/tests/test_init_alloc_failure.py new file mode 100644 index 00000000..fe3a732c --- /dev/null +++ b/tests/test_init_alloc_failure.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[1] +library = next((root / "librhash").glob("librhash.so.*")) +with tempfile.TemporaryDirectory() as directory: + directory = Path(directory) + hook = directory / "fail_alloc.c" + driver = directory / "check_init.c" + hook.write_text(r'''#define _GNU_SOURCE +#include +#include +#include +static int fail_next; +static int failed_count; +static void* (*real_aligned_alloc)(size_t, size_t); +__attribute__((constructor)) static void resolve_aligned_alloc(void) { + real_aligned_alloc = dlsym(RTLD_NEXT, "aligned_alloc"); +} +void rhash_test_fail_next_aligned_alloc(void) { fail_next = 1; } +int rhash_test_failed_count(void) { return failed_count; } +void* aligned_alloc(size_t alignment, size_t size) { + if (fail_next) { fail_next = 0; failed_count++; errno = ENOMEM; return NULL; } + return real_aligned_alloc(alignment, size); +} +''') + driver.write_text(r'''#include +#include +#include +int main(void) { + unsigned id = RHASH_SHA256; + void (*fail_next)(void) = dlsym(RTLD_DEFAULT, "rhash_test_fail_next_aligned_alloc"); + int (*failed_count)(void) = dlsym(RTLD_DEFAULT, "rhash_test_failed_count"); + rhash ctx; + if (!fail_next || !failed_count) return 2; + rhash_library_init(); + fail_next(); + ctx = rhash_init_multi(1, &id); + if (ctx) { rhash_free(ctx); return 3; } + if (failed_count() != 1) return 4; + puts("PASS: allocation failure returns NULL"); + return 0; +} +''') + cc = os.environ.get("CC", "cc") + subprocess.run([cc, "-shared", "-fPIC", str(hook), "-ldl", "-o", str(directory / "fail_alloc.so")], check=True) + subprocess.run([cc, str(driver), "-I", str(root / "librhash"), str(library), "-ldl", + f"-Wl,-rpath,{root / 'librhash'}", "-o", str(directory / "check_init")], check=True) + env = dict(os.environ, LD_PRELOAD=str(directory / "fail_alloc.so"), + LD_LIBRARY_PATH=str(root / "librhash"), + UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") + result = subprocess.run([str(directory / "check_init")], env=env, capture_output=True, + text=True, timeout=10) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert "PASS: allocation failure returns NULL" in result.stdout, result.stdout + assert "runtime error:" not in result.stderr, result.stderr +print("PASS: rhash_init_multi handles aligned-allocation failure") From b59f5f326a0934bc89f2cbe4fd6435535867fe1b Mon Sep 17 00:00:00 2001 From: Aleksey Kravchenko Date: Mon, 19 Oct 2026 03:02:38 +0300 Subject: [PATCH 2/2] Delete tests/test_init_alloc_failure.py --- tests/test_init_alloc_failure.py | 60 -------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 tests/test_init_alloc_failure.py diff --git a/tests/test_init_alloc_failure.py b/tests/test_init_alloc_failure.py deleted file mode 100644 index fe3a732c..00000000 --- a/tests/test_init_alloc_failure.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -import os -from pathlib import Path -import subprocess -import tempfile - -root = Path(__file__).resolve().parents[1] -library = next((root / "librhash").glob("librhash.so.*")) -with tempfile.TemporaryDirectory() as directory: - directory = Path(directory) - hook = directory / "fail_alloc.c" - driver = directory / "check_init.c" - hook.write_text(r'''#define _GNU_SOURCE -#include -#include -#include -static int fail_next; -static int failed_count; -static void* (*real_aligned_alloc)(size_t, size_t); -__attribute__((constructor)) static void resolve_aligned_alloc(void) { - real_aligned_alloc = dlsym(RTLD_NEXT, "aligned_alloc"); -} -void rhash_test_fail_next_aligned_alloc(void) { fail_next = 1; } -int rhash_test_failed_count(void) { return failed_count; } -void* aligned_alloc(size_t alignment, size_t size) { - if (fail_next) { fail_next = 0; failed_count++; errno = ENOMEM; return NULL; } - return real_aligned_alloc(alignment, size); -} -''') - driver.write_text(r'''#include -#include -#include -int main(void) { - unsigned id = RHASH_SHA256; - void (*fail_next)(void) = dlsym(RTLD_DEFAULT, "rhash_test_fail_next_aligned_alloc"); - int (*failed_count)(void) = dlsym(RTLD_DEFAULT, "rhash_test_failed_count"); - rhash ctx; - if (!fail_next || !failed_count) return 2; - rhash_library_init(); - fail_next(); - ctx = rhash_init_multi(1, &id); - if (ctx) { rhash_free(ctx); return 3; } - if (failed_count() != 1) return 4; - puts("PASS: allocation failure returns NULL"); - return 0; -} -''') - cc = os.environ.get("CC", "cc") - subprocess.run([cc, "-shared", "-fPIC", str(hook), "-ldl", "-o", str(directory / "fail_alloc.so")], check=True) - subprocess.run([cc, str(driver), "-I", str(root / "librhash"), str(library), "-ldl", - f"-Wl,-rpath,{root / 'librhash'}", "-o", str(directory / "check_init")], check=True) - env = dict(os.environ, LD_PRELOAD=str(directory / "fail_alloc.so"), - LD_LIBRARY_PATH=str(root / "librhash"), - UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1") - result = subprocess.run([str(directory / "check_init")], env=env, capture_output=True, - text=True, timeout=10) - assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) - assert "PASS: allocation failure returns NULL" in result.stdout, result.stdout - assert "runtime error:" not in result.stderr, result.stderr -print("PASS: rhash_init_multi handles aligned-allocation failure")