From d42898c51e260b5e87173d20df60c2c3df1aaa41 Mon Sep 17 00:00:00 2001 From: Ted Poole Date: Tue, 11 Aug 2026 17:58:57 +0100 Subject: [PATCH] compat/openssl: Ensure OpenSSL uses tcmalloc functions When dynamically loading the OpenSSL shared libraries, we use the RTLD_DEEPBIND option so that the compatibility layer's wrapper symbols don't shadow the real OpenSSL ones. As an unintended consequence of this, it causes OpenSSL's calls to malloc/realloc/free to bind to glibc's allocator instead of the tcmalloc that is statically linked into Envoy. The result is that OpenSSL operates on a completely separate heap, defeating tcmalloc's performance benefits on the TLS hot path and making all OpenSSL allocations invisible to tcmalloc heap profiling and memory dumps. This is fixed by calling CRYPTO_set_mem_functions() immediately after loading libcrypto.so, thus redirecting OpenSSL's malloc/realloc/free calls to the main executable's tcmalloc functions. Also added a test to verify that OpenSSL is using allocator functions that reside in the main executable (tcmalloc) rather than inside `libcrypto.so` (glibc via `RTLD_DEEPBIND`), preventing regressions. Backport of envoyproxy/envoy#46562 & envoyproxy/envoy#46576 Signed-off-by: Ted Poole Signed-off-by: Jonh Wendell Co-authored-by: Claude Opus 4.6 --- bssl-compat/prefixer/prefixer.cpp | 44 ++++++++++++++++--- bssl-compat/source/test/test_crypto.cc | 20 +++++++++ .../tls__ensure-openssl-uses-tcmalloc.rst | 4 ++ 3 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 changelogs/current/bug_fixes/tls__ensure-openssl-uses-tcmalloc.rst diff --git a/bssl-compat/prefixer/prefixer.cpp b/bssl-compat/prefixer/prefixer.cpp index 4c2f7253bae..5e785a688b9 100644 --- a/bssl-compat/prefixer/prefixer.cpp +++ b/bssl-compat/prefixer/prefixer.cpp @@ -491,6 +491,24 @@ void MyFrontendAction::EndSourceFileAction() { << "static void " << opt::prefix << "_init(void) __attribute__ ((constructor));" << std::endl << "static void " << opt::prefix << "_fini(void) __attribute__ ((destructor));" << std::endl << std::endl + << "static void *ossl_malloc(size_t num, const char *file, int line) {" << std::endl + << " (void)file;" << std::endl + << " (void)line;" << std::endl + << " return malloc(num);" << std::endl + << "}" << std::endl + << std::endl + << "static void *ossl_realloc(void *addr, size_t num, const char *file, int line) {" << std::endl + << " (void)file;" << std::endl + << " (void)line;" << std::endl + << " return realloc(addr, num);" << std::endl + << "}" << std::endl + << std::endl + << "static void ossl_free(void *addr, const char *file, int line) {" << std::endl + << " (void)file;" << std::endl + << " (void)line;" << std::endl + << " free(addr);" << std::endl + << "}" << std::endl + << std::endl << "static void *lookup(const char *symbol) {" << std::endl << " void *result;" << std::endl << " const char *s = symbol + " << opt::prefix.size() + 1 << ";" << std::endl @@ -510,11 +528,6 @@ void MyFrontendAction::EndSourceFileAction() { << " exit(ELIBACC);" << std::endl << " }" << std::endl << std::endl - << " if((libssl = ossl_dlopen(LIBSSL_SO)) == NULL) {" << std::endl - << " fprintf(stderr, \"%s: dlopen(%s) : %s\\n\", __func__, LIBSSL_SO, dlerror());" << std::endl - << " exit(ELIBACC);" << std::endl - << " }" << std::endl - << std::endl << " ossl.ossl_OpenSSL_version_num = (ossl_OpenSSL_version_num_t)lookup(\"ossl_OpenSSL_version_num\");" << std::endl << " if (ossl.ossl_OpenSSL_version_num == NULL) {" << std::endl << " fprintf(stderr, \"%s: Failed to load OpenSSL_version_num()\\n\", __func__);" << std::endl @@ -533,6 +546,27 @@ void MyFrontendAction::EndSourceFileAction() { << " major, minor, patch);" << std::endl << " exit(ELIBACC);" << std::endl << " }" << std::endl + << std::endl + << " typedef int (*CRYPTO_set_mem_functions_fn)(" << std::endl + << " void *(*malloc_fn)(size_t, const char *, int)," << std::endl + << " void *(*realloc_fn)(void *, size_t, const char *, int)," << std::endl + << " void (*free_fn)(void *, const char *, int));" << std::endl + << std::endl + << " CRYPTO_set_mem_functions_fn set_mem_fn = lookup(\"CRYPTO_set_mem_functions\");" << std::endl + << " if (set_mem_fn == NULL) {" << std::endl + << " fprintf(stderr, \"%s: dlsym(libcrypto, \\\"CRYPTO_set_mem_functions\\\") : %s\\n\", __func__, dlerror());" << std::endl + << " exit(ELIBACC);" << std::endl + << " }" << std::endl + << std::endl + << " if (!set_mem_fn(ossl_malloc, ossl_realloc, ossl_free)) {" << std::endl + << " fprintf(stderr, \"%s: CRYPTO_set_mem_functions() failed\\n\", __func__);" << std::endl + << " exit(ELIBACC);" << std::endl + << " }" << std::endl + << std::endl + << " if((libssl = ossl_dlopen(LIBSSL_SO)) == NULL) {" << std::endl + << " fprintf(stderr, \"%s: dlopen(%s) : %s\\n\", __func__, LIBSSL_SO, dlerror());" << std::endl + << " exit(ELIBACC);" << std::endl + << " }" << std::endl << std::endl; for(const auto &function : m_functions) { diff --git a/bssl-compat/source/test/test_crypto.cc b/bssl-compat/source/test/test_crypto.cc index e4a882154e2..b87c394f3dc 100644 --- a/bssl-compat/source/test/test_crypto.cc +++ b/bssl-compat/source/test/test_crypto.cc @@ -5,3 +5,23 @@ TEST(TestCrypto, test_FIPS_mode) { ASSERT_EQ(0, FIPS_mode()); } +#ifdef BSSL_COMPAT +#include + +TEST(TestCrypto, test_openssl_uses_main_executable_allocator) { + ossl_CRYPTO_malloc_fn malloc_fn = nullptr; + ossl_CRYPTO_realloc_fn realloc_fn = nullptr; + ossl_CRYPTO_free_fn free_fn = nullptr; + ossl_CRYPTO_get_mem_functions(&malloc_fn, &realloc_fn, &free_fn); + + Dl_info info; + ASSERT_NE(dladdr(reinterpret_cast(malloc_fn), &info), 0); + EXPECT_EQ(std::string(info.dli_fname).find("libcrypto"), std::string::npos); + + ASSERT_NE(dladdr(reinterpret_cast(realloc_fn), &info), 0); + EXPECT_EQ(std::string(info.dli_fname).find("libcrypto"), std::string::npos); + + ASSERT_NE(dladdr(reinterpret_cast(free_fn), &info), 0); + EXPECT_EQ(std::string(info.dli_fname).find("libcrypto"), std::string::npos); +} +#endif \ No newline at end of file diff --git a/changelogs/current/bug_fixes/tls__ensure-openssl-uses-tcmalloc.rst b/changelogs/current/bug_fixes/tls__ensure-openssl-uses-tcmalloc.rst new file mode 100644 index 00000000000..c5ae1c89ea8 --- /dev/null +++ b/changelogs/current/bug_fixes/tls__ensure-openssl-uses-tcmalloc.rst @@ -0,0 +1,4 @@ +Fixed a bug where OpenSSL was using glibc's allocator instead of tcmalloc. This +resulted in OpenSSL operating on a completely separate heap, defeating +tcmalloc's performance benefits on the TLS hot path and making all OpenSSL +allocations invisible to tcmalloc heap profiling and memory dumps.