Repository navigation
test: thread lifecycle × profiler-state coverage #624
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
4769512
test: add thread lifecycle × profiler-state coverage tests
jbachorik b31f270
review: fix catch type and clarify synthetic test comment
jbachorik 319a518
Potential fix for pull request finding 'Ignored error status of call'
jbachorik c704666
Potential fix for pull request finding 'Ignored error status of call'
jbachorik 76b61ab
test: make lifecycle fuzz tests more aggressive
jbachorik 536d6bb
Merge branch 'main' into lifecycle-fuzzing-tests
jbachorik 4c6a5e1
fix: copyright headers, latch-timeout cleanup, remove no-op @RetryTest
jbachorik d4debf9
fix: disable thread filter and propagate churn worker failures in lif…
jbachorik 5216eff
fix: report all churn/profiler race errors via composite AssertionError
jbachorik 52864e4
fix: apply composite AssertionError to StopWithChurningThreadsTest too
jbachorik File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| /* | ||
| * Copyright 2026, Datadog, Inc. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * Coverage for three previously-untested cells in the thread-lifecycle × | ||
| * profiler-state matrix, exercised at the native layer (no JVM required): | ||
| * | ||
| * Cell 1 – Disabling + teardown: ThreadFilter::unregisterThread races the | ||
| * clearActive() drain that stop() performs before disabling. | ||
| * Cell 2 – Enabling + startup: ThreadFilter::registerThread races a | ||
| * concurrent init() call that alternately enables/disables the filter. | ||
| * Cell 3 – Disabled + teardown: unregisterThread(-1) called when the filter | ||
| * was never initialised (idle profiler path). | ||
| * | ||
| * All three test cases are runnable under TSan and ASan. | ||
| */ | ||
| #include "gtest/gtest.h" | ||
|
|
||
| #ifdef __linux__ | ||
|
|
||
| #include "threadFilter.h" | ||
| #include "thread.h" | ||
| #include "../../main/cpp/gtest_crash_handler.h" | ||
|
|
||
| #include <atomic> | ||
| #include <thread> | ||
| #include <vector> | ||
|
|
||
| static constexpr const char TEST_NAME[] = "ThreadFilterLifecycle"; | ||
| static constexpr int kWorkers = 16; | ||
| static constexpr int kIterations = 2000; | ||
|
|
||
| // ── Cell 1: unregisterThread racing clearActive (simulated stop() drain) ────── | ||
|
|
||
| TEST(ThreadFilterLifecycle, UnregisterRacesClearActive) { | ||
| installGtestCrashHandler<TEST_NAME>(); | ||
|
|
||
| ThreadFilter filter; | ||
| filter.init("*"); | ||
| ASSERT_TRUE(filter.enabled()); | ||
|
|
||
| std::atomic<bool> run{true}; | ||
|
|
||
| // Drain thread: simulates what stop() does — clearActive() empties the active | ||
| // slots without disabling the filter, to flush in-flight samples. | ||
| std::thread drainer([&]() { | ||
| while (run.load(std::memory_order_relaxed)) { | ||
| filter.clearActive(); | ||
| } | ||
| }); | ||
|
|
||
| std::vector<std::thread> workers; | ||
| workers.reserve(kWorkers); | ||
| for (int i = 0; i < kWorkers; i++) { | ||
| workers.emplace_back([&]() { | ||
| for (int j = 0; j < kIterations; j++) { | ||
| ProfiledThread::initCurrentThread(); | ||
| ProfiledThread* self = ProfiledThread::current(); | ||
| if (!self) { | ||
| ProfiledThread::release(); | ||
| continue; | ||
| } | ||
| ThreadFilter::SlotID slot = filter.registerThread(); | ||
| if (slot >= 0) { | ||
| filter.add(self->tid(), slot); | ||
| filter.remove(slot); | ||
| filter.unregisterThread(slot); | ||
| self->setFilterSlotId(-1); | ||
| } | ||
| ProfiledThread::release(); | ||
| } | ||
| }); | ||
| } | ||
| for (auto& t : workers) t.join(); | ||
| run.store(false, std::memory_order_relaxed); | ||
| drainer.join(); | ||
|
|
||
| restoreDefaultSignalHandlers(); | ||
| SUCCEED(); | ||
| } | ||
|
|
||
| // ── Cell 2: registerThread racing concurrent init() enable/disable toggles ──── | ||
|
|
||
| TEST(ThreadFilterLifecycle, RegisterRacesInit) { | ||
| installGtestCrashHandler<TEST_NAME>(); | ||
|
|
||
| ThreadFilter filter; | ||
|
|
||
| std::atomic<bool> run{true}; | ||
|
|
||
| // Init thread: alternates between enabling ("*") and disabling (nullptr) the | ||
| // filter; this is a synthetic enable/disable stress on _enabled, not a | ||
| // faithful replay of start()/stop() (production stop() never calls init()). | ||
| std::thread initer([&]() { | ||
| int toggle = 0; | ||
| while (run.load(std::memory_order_relaxed)) { | ||
| // Even: enable ("*"), odd: disable (nullptr / empty string) | ||
| filter.init(toggle++ % 2 == 0 ? "*" : nullptr); | ||
| } | ||
| }); | ||
|
|
||
| std::vector<std::thread> workers; | ||
| workers.reserve(kWorkers); | ||
| for (int i = 0; i < kWorkers; i++) { | ||
| workers.emplace_back([&]() { | ||
| for (int j = 0; j < kIterations; j++) { | ||
| // Mirror the production onThreadStart guard: skip if filter disabled. | ||
| if (!filter.enabled()) continue; | ||
|
|
||
| ProfiledThread::initCurrentThread(); | ||
| ProfiledThread* self = ProfiledThread::current(); | ||
| if (!self) { | ||
| ProfiledThread::release(); | ||
| continue; | ||
| } | ||
| ThreadFilter::SlotID slot = filter.registerThread(); | ||
| if (slot >= 0) { | ||
| // filter may have been disabled between the enabled() check and | ||
| // registerThread(); the slot is still valid and must be released. | ||
| filter.unregisterThread(slot); | ||
| self->setFilterSlotId(-1); | ||
| } | ||
| ProfiledThread::release(); | ||
| } | ||
| }); | ||
| } | ||
| for (auto& t : workers) t.join(); | ||
| run.store(false, std::memory_order_relaxed); | ||
| initer.join(); | ||
|
|
||
| restoreDefaultSignalHandlers(); | ||
| SUCCEED(); | ||
| } | ||
|
|
||
| // ── Cell 3: unregisterThread with sentinel slot when filter never init'd ─────── | ||
|
|
||
| TEST(ThreadFilterLifecycle, UnregisterWithNoInitDoesNotCrash) { | ||
| // No crash handler — single-threaded, expected outcome is "no crash". | ||
| ThreadFilter filter; | ||
|
|
||
| // enabled() must be false without init(). | ||
| EXPECT_FALSE(filter.enabled()); | ||
|
|
||
| // unregisterThread(-1) is what onThreadEnd calls when filterSlotId() == -1 | ||
| // (thread was registered before profiler started, so slot was never assigned). | ||
| // Must not crash, assert, or invoke UB. | ||
| filter.unregisterThread(-1); | ||
|
|
||
| SUCCEED(); | ||
| } | ||
|
|
||
| #endif // __linux__ |
64 changes: 64 additions & 0 deletions
64
ddprof-test/src/test/java/com/datadoghq/profiler/lifecycle/IdleThreadLifecycleTest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| /* | ||
| * Copyright 2026, Datadog, Inc. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package com.datadoghq.profiler.lifecycle; | ||
|
|
||
| import com.datadoghq.profiler.JavaProfiler; | ||
| import com.datadoghq.profiler.Platform; | ||
|
|
||
| import org.junit.jupiter.api.Assumptions; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.junit.jupiter.api.Timeout; | ||
|
|
||
| import java.util.concurrent.CountDownLatch; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertFalse; | ||
| import static org.junit.jupiter.api.Assertions.assertTrue; | ||
|
|
||
| /** | ||
| * Regression test for the "Disabled + teardown" coverage cell. | ||
| * | ||
| * <p>Verifies that thread lifecycle events (ThreadStart/ThreadEnd JVMTI callbacks, | ||
| * ProfiledThread::release) are safe when the profiler has never been started. | ||
| * The JVMTI callbacks registered in JNI_OnLoad delegate to onThreadStart / | ||
| * onThreadEnd unconditionally; those paths must tolerate a never-started profiler. | ||
| * | ||
| * <p>Does NOT call profiler.execute() — the profiler stays idle throughout. | ||
| */ | ||
| public class IdleThreadLifecycleTest { | ||
|
|
||
| private static final int THREAD_COUNT = 200; | ||
|
|
||
| @Timeout(30) | ||
| @Test | ||
| public void threadLifecycleWhileProfilerIdle() throws Exception { | ||
| Assumptions.assumeTrue(!Platform.isJ9(), "J9 has a different JVMTI implementation"); | ||
| Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); | ||
|
|
||
| JavaProfiler profiler = JavaProfiler.getInstance(); | ||
| // Profiler must NOT be started — verify it is idle before we begin. | ||
| String statusBefore = profiler.getStatus(); | ||
| assertFalse(statusBefore.contains("Running : true"), | ||
| "Profiler must be idle at test start; status: " + statusBefore); | ||
|
|
||
| ExecutorService pool = Executors.newFixedThreadPool(16); | ||
| CountDownLatch done = new CountDownLatch(THREAD_COUNT); | ||
| for (int i = 0; i < THREAD_COUNT; i++) { | ||
| pool.submit(done::countDown); | ||
| } | ||
| assertTrue(done.await(10, TimeUnit.SECONDS), | ||
| "Not all " + THREAD_COUNT + " thread tasks completed"); | ||
| pool.shutdown(); | ||
| pool.awaitTermination(5, TimeUnit.SECONDS); | ||
|
|
||
| // Profiler must still be idle — thread lifecycle events must not trigger a start. | ||
| String statusAfter = profiler.getStatus(); | ||
| assertFalse(statusAfter.contains("Running : true"), | ||
| "Profiler must remain idle after thread churn; status: " + statusAfter); | ||
| } | ||
| } | ||
135 changes: 135 additions & 0 deletions
135
ddprof-test/src/test/java/com/datadoghq/profiler/lifecycle/StartWithChurningThreadsTest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| /* | ||
| * Copyright 2026, Datadog, Inc. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package com.datadoghq.profiler.lifecycle; | ||
|
|
||
| import com.datadoghq.profiler.JavaProfiler; | ||
| import com.datadoghq.profiler.Platform; | ||
|
|
||
| import org.junit.jupiter.api.Assumptions; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.junit.jupiter.api.Timeout; | ||
|
|
||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Queue; | ||
| import java.util.concurrent.CancellationException; | ||
| import java.util.concurrent.CountDownLatch; | ||
| import java.util.concurrent.ExecutionException; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.Future; | ||
| import java.util.concurrent.LinkedBlockingQueue; | ||
| import java.util.concurrent.TimeUnit; | ||
| import java.util.concurrent.atomic.AtomicBoolean; | ||
|
|
||
| import static org.junit.jupiter.api.Assertions.assertTrue; | ||
|
|
||
| /** | ||
| * Regression test for the "Enabling + startup/teardown" coverage cell. | ||
| * | ||
| * <p>Races profiler.execute() against threads that are already generating JVMTI | ||
| * ThreadStart/ThreadEnd events. The hazard window: execute() installs signal | ||
| * handlers, initialises the thread filter, and registers JVMTI callbacks one by | ||
| * one; a thread starting or ending during this sequence may call onThreadStart / | ||
| * onThreadEnd on a half-initialised engine. | ||
| * | ||
| * <p>Does NOT extend AbstractProfilerTest — needs direct control over | ||
| * execute()/stop() timing; uses the same pattern as ShutdownTest. | ||
| */ | ||
| public class StartWithChurningThreadsTest { | ||
|
|
||
| private static final int CHURN_THREADS = 64; | ||
| private static final int CYCLES = 500; | ||
| // filter= disables thread filtering so all threads receive SIGVTALRM, | ||
| // maximising the chance of racing execute() with active thread churn. | ||
| private static final String PROFILER_CMD = "start,wall=1ms,filter=,jfr,file="; | ||
|
|
||
| @Timeout(120) | ||
| @Test | ||
| public void startRacesThreadStartEnd() throws Exception { | ||
| Assumptions.assumeTrue(!Platform.isJ9(), "J9 has a different JVMTI implementation"); | ||
| Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11), "wall-clock requires Java 11+"); | ||
| Assumptions.assumeTrue(!Platform.isZing(), "Zing forces cstack=no"); | ||
|
|
||
| JavaProfiler profiler = JavaProfiler.getInstance(); | ||
| Path recordings = Files.createTempDirectory("lifecycle-cell2-"); | ||
| Queue<Throwable> errors = new LinkedBlockingQueue<>(); | ||
|
|
||
| AtomicBoolean running = new AtomicBoolean(true); | ||
| // Latch: churn threads signal they are running before execute() is called, | ||
| // ensuring JVMTI events fire during profiler initialisation. | ||
| CountDownLatch churnRunning = new CountDownLatch(CHURN_THREADS); | ||
| ExecutorService churn = Executors.newFixedThreadPool(CHURN_THREADS); | ||
| List<Future<?>> churnFutures = new ArrayList<>(CHURN_THREADS); | ||
|
|
||
| for (int i = 0; i < CHURN_THREADS; i++) { | ||
| churnFutures.add(churn.submit(() -> { | ||
| churnRunning.countDown(); | ||
| while (running.get()) { | ||
| new Thread(() -> {}).start(); | ||
| } | ||
| })); | ||
| } | ||
|
|
||
| try { | ||
| // Wait for churn to be active BEFORE calling execute() | ||
| assertTrue(churnRunning.await(10, TimeUnit.SECONDS), "Churn threads did not start"); | ||
| for (int cycle = 0; cycle < CYCLES; cycle++) { | ||
| Path jfr = Files.createTempFile(recordings, "c2-" + cycle + "-", ".jfr"); | ||
| try { | ||
| // execute() races with active thread churn | ||
| profiler.execute(PROFILER_CMD + jfr.toAbsolutePath()); | ||
| Thread.sleep(cycle % 3); | ||
| profiler.stop(); | ||
| } catch (Throwable t) { | ||
| errors.add(t); | ||
| break; | ||
| } finally { | ||
| Files.deleteIfExists(jfr); | ||
| } | ||
| } | ||
| } finally { | ||
| running.set(false); | ||
| churn.shutdown(); | ||
| if (!churn.awaitTermination(5, TimeUnit.SECONDS)) { | ||
| churn.shutdownNow(); | ||
| } | ||
| try { | ||
| profiler.stop(); | ||
| } catch (Exception ignored) { | ||
| // already stopped | ||
| } | ||
| deleteDirectory(recordings); | ||
| for (Future<?> f : churnFutures) { | ||
| try { | ||
| f.get(); | ||
| } catch (ExecutionException e) { | ||
| errors.add(e.getCause()); | ||
| } catch (CancellationException ignored) {} | ||
| } | ||
| } | ||
|
|
||
| if (!errors.isEmpty()) { | ||
| AssertionError composite = new AssertionError( | ||
| errors.size() + " error(s) occurred during churn/profiler race"); | ||
| for (Throwable t : errors) { | ||
| composite.addSuppressed(t); | ||
| } | ||
| throw composite; | ||
| } | ||
| } | ||
|
|
||
| private static void deleteDirectory(Path dir) { | ||
| try { | ||
| Files.walk(dir) | ||
| .sorted(java.util.Comparator.reverseOrder()) | ||
| .forEach(p -> { try { Files.deleteIfExists(p); } catch (IOException ignored) {} }); | ||
| } catch (IOException ignored) {} | ||
| } | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.