From e1f0a784a0ba5754489c6607841a8b4f677d9171 Mon Sep 17 00:00:00 2001 From: Richard Tingle <6330028+richardTingle@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:20:32 +0000 Subject: [PATCH] Introduce support for multi-scenario screenshot tests to validate identical results across approaches. --- .../testframework/Scenario.java | 13 ++ .../testframework/ScreenshotTest.java | 14 +- .../testframework/ScreenshotTestBase.java | 14 +- .../testframework/TestDriver.java | 166 +++++++++++------- 4 files changed, 144 insertions(+), 63 deletions(-) create mode 100644 jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/Scenario.java diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/Scenario.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/Scenario.java new file mode 100644 index 0000000000..64b0141ee4 --- /dev/null +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/Scenario.java @@ -0,0 +1,13 @@ +package org.jmonkeyengine.screenshottests.testframework; + +import com.jme3.app.state.AppState; + +public class Scenario { + String scenarioName; + AppState[] states; + + public Scenario(String scenarioName, AppState... states) { + this.scenarioName = scenarioName; + this.states = states; + } +} diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java index 5bff5d9960..f207e73d27 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTest.java @@ -47,7 +47,11 @@ public class ScreenshotTest{ TestType testType = TestType.MUST_PASS; - AppState[] states; + /** + * Usually there will be a single scenario but sometimes it will be desirable to test that two ways + * of doing something produce the same result. In that case there will be multiple scenarios. + */ + List scenarios = new ArrayList<>(); List framesToTakeScreenshotsOn = new ArrayList<>(); @@ -56,7 +60,11 @@ public class ScreenshotTest{ String baseImageFileName = null; public ScreenshotTest(AppState... initialStates){ - states = initialStates; + scenarios.add(new Scenario("SimpleSingleScenario", initialStates)); + framesToTakeScreenshotsOn.add(1); //default behaviour is to take a screenshot on the first frame + } + public ScreenshotTest(Scenario... scenarios){ + this.scenarios.addAll(Arrays.asList(scenarios)); framesToTakeScreenshotsOn.add(1); //default behaviour is to take a screenshot on the first frame } @@ -100,7 +108,7 @@ public void run(){ String imageFilePrefix = baseImageFileName == null ? calculateImageFilePrefix() : baseImageFileName; - TestDriver.bootAppForTest(testType,settings,imageFilePrefix, framesToTakeScreenshotsOn, states); + TestDriver.bootAppForTest(testType,settings,imageFilePrefix, framesToTakeScreenshotsOn, scenarios); } diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTestBase.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTestBase.java index 81553e614e..147946e161 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTestBase.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/ScreenshotTestBase.java @@ -47,10 +47,22 @@ public abstract class ScreenshotTestBase{ /** * Initialises a screenshot test. The resulting object should be configured (if neccessary) and then started * by calling {@link ScreenshotTest#run()}. - * @param initialStates + * @param initialStates the states that will create the JME environment * @return */ public ScreenshotTest screenshotTest(AppState... initialStates){ return new ScreenshotTest(initialStates); } + + /** + * Permits multiple scenarios to be tested in a single test. Each scenario should give identical results and + * will have a screenshot taken on the same frame. + * + *

+ * This is intended for testing migrations where the old and new approach should both give identical results. + *

+ */ + public ScreenshotTest screenshotMultiScenarioTest(Scenario... scenarios){ + return new ScreenshotTest(scenarios); + } } diff --git a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java index ca0cfa98a9..2d7220de14 100644 --- a/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java +++ b/jme3-screenshot-tests/src/main/java/org/jmonkeyengine/screenshottests/testframework/TestDriver.java @@ -56,7 +56,9 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.Executors; @@ -78,7 +80,9 @@ public class TestDriver extends BaseAppState{ private static final Logger logger = Logger.getLogger(TestDriver.class.getName()); - public static final String IMAGES_ARE_DIFFERENT = "Images are different. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; + public static final String IMAGES_ARE_DIFFERENT = "Generated images is different from committed image. (If you are running the test locally this is expected, images only reproducible on github CI infrastructure)"; + + public static final String IMAGES_ARE_DIFFERENT_BETWEEN_SCENARIOS = "Images are different between scenarios."; public static final String IMAGES_ARE_DIFFERENT_SIZES = "Images are different sizes."; @@ -145,85 +149,127 @@ public void update(float tpf){ * - After all the frames have been taken it stops the application * - Compares the screenshot to the expected screenshot (if any). Fails the test if they are different */ - public static void bootAppForTest(TestType testType, AppSettings appSettings, String baseImageFileName, List framesToTakeScreenshotsOn, AppState... initialStates){ - FastMath.rand.setSeed(0); //try to make things deterministic by setting the random seed + public static void bootAppForTest(TestType testType, AppSettings appSettings, String baseImageFileName, List framesToTakeScreenshotsOn, List scenarios){ + Collections.sort(framesToTakeScreenshotsOn); - Path imageTempDir; + List tempFolders = new ArrayList<>(); + Map> imageFilesPerScenario = new HashMap<>(); + + // usually there is a single scenario, but the framework can be set up to expect multiple scenarios that give identical results + for(Scenario scenario : scenarios) { + FastMath.rand.setSeed(0); //try to make things deterministic by setting the random seed + Path imageTempDir; + try { + imageTempDir = Files.createTempDirectory("jmeSnapshotTest"); + } catch (IOException e) { + throw new RuntimeException(e); + } + tempFolders.add(imageTempDir); - try{ - imageTempDir = Files.createTempDirectory("jmeSnapshotTest"); - } catch(IOException e){ - throw new RuntimeException(e); - } + ScreenshotNoInputAppState screenshotAppState = new ScreenshotNoInputAppState(imageTempDir.toString() + "/"); + String screenshotAppFileNamePrefix = "Screenshot-"; + screenshotAppState.setFileName(screenshotAppFileNamePrefix); - ScreenshotNoInputAppState screenshotAppState = new ScreenshotNoInputAppState(imageTempDir.toString() + "/"); - String screenshotAppFileNamePrefix = "Screenshot-"; - screenshotAppState.setFileName(screenshotAppFileNamePrefix); + List states = new ArrayList<>(Arrays.asList(scenario.states)); + TestDriver testDriver = new TestDriver(screenshotAppState, framesToTakeScreenshotsOn); + states.add(screenshotAppState); + states.add(testDriver); - List states = new ArrayList<>(Arrays.asList(initialStates)); - TestDriver testDriver = new TestDriver(screenshotAppState, framesToTakeScreenshotsOn); - states.add(screenshotAppState); - states.add(testDriver); + SimpleApplication app = new App(states.toArray(new AppState[0])); + app.setSettings(appSettings); + app.setShowSettings(false); - SimpleApplication app = new App(states.toArray(new AppState[0])); - app.setSettings(appSettings); - app.setShowSettings(false); + testDriver.waitLatch = new CountDownLatch(1); + executor.execute(() -> app.start(JmeContext.Type.Display)); - testDriver.waitLatch = new CountDownLatch(1); - executor.execute(() -> app.start(JmeContext.Type.Display)); + int maxWaitTimeMilliseconds = 45000; - int maxWaitTimeMilliseconds = 45000; + try { + boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); - try { - boolean exitedProperly = testDriver.waitLatch.await(maxWaitTimeMilliseconds, TimeUnit.MILLISECONDS); + if (!exitedProperly) { + logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); + app.stop(true); + } - if(!exitedProperly){ - logger.warning("Test driver did not exit in " + maxWaitTimeMilliseconds + "ms. Timed out"); - app.stop(true); + Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this) + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); } - Thread.sleep(1000); //give time for openGL is fully released before starting a new test (get random JVM crashes without this) - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } + //search the imageTempDir + List imageFiles = new ArrayList<>(); + try (Stream paths = Files.list(imageTempDir)) { + paths.forEach(imageFiles::add); + } catch (IOException e) { + throw new RuntimeException(e); + } - //search the imageTempDir - List imageFiles = new ArrayList<>(); - try(Stream paths = Files.list(imageTempDir)){ - paths.forEach(imageFiles::add); - } catch(IOException e){ - throw new RuntimeException(e); - } + //this resorts with natural numeric ordering (so App10.png comes after App9.png) + imageFiles.sort(new Comparator() { + @Override + public int compare(Path p1, Path p2) { + return extractNumber(p1).compareTo(extractNumber(p2)); + } - //this resorts with natural numeric ordering (so App10.png comes after App9.png) - imageFiles.sort(new Comparator(){ - @Override - public int compare(Path p1, Path p2){ - return extractNumber(p1).compareTo(extractNumber(p2)); + private Integer extractNumber(Path path) { + String name = path.getFileName().toString(); + int numStart = screenshotAppFileNamePrefix.length(); + int numEnd = name.lastIndexOf(".png"); + return Integer.parseInt(name.substring(numStart, numEnd)); + } + }); + if (imageFiles.isEmpty()) { + fail("No screenshot found in the temporary directory. Did the application crash?"); } - - private Integer extractNumber(Path path){ - String name = path.getFileName().toString(); - int numStart = screenshotAppFileNamePrefix.length(); - int numEnd = name.lastIndexOf(".png"); - return Integer.parseInt(name.substring(numStart, numEnd)); + if (imageFiles.size() != framesToTakeScreenshotsOn.size()) { + fail("Not all screenshots were taken, expected " + framesToTakeScreenshotsOn.size() + " but got " + imageFiles.size()); } - }); - if(imageFiles.isEmpty()){ - fail("No screenshot found in the temporary directory. Did the application crash?"); + imageFilesPerScenario.put(scenario, imageFiles); } - if(imageFiles.size() != framesToTakeScreenshotsOn.size()){ - fail("Not all screenshots were taken, expected " + framesToTakeScreenshotsOn.size() + " but got " + imageFiles.size()); - } - String failureMessage = null; try { + List primeScenarioScreenshots = imageFilesPerScenario.get(scenarios.get(0)); + + if(imageFilesPerScenario.size()>1){ + String primeScenarioName = scenarios.get(0).scenarioName; + + // check each scenario gave the same results (before checking a single scenario against the reference images + for(int i=1;i otherScenarioScreenshots = imageFilesPerScenario.get(scenarios.get(i)); + for(int screenshotIndex=0;screenshotIndex