diff --git a/src/main/java/org/apache/groovy/runtime/indy/AotDispatch.java b/src/main/java/org/apache/groovy/runtime/indy/AotDispatch.java new file mode 100644 index 00000000000..5a1e196d05d --- /dev/null +++ b/src/main/java/org/apache/groovy/runtime/indy/AotDispatch.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.groovy.runtime.indy; + +import java.lang.invoke.SwitchPoint; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Ahead-of-time link mode for indy dispatch (incubating). + *

+ * GraalVM native image supports every {@code java.lang.invoke} building block Groovy's indy + * runtime uses except retargeting an existing call site: both + * {@code MutableCallSite.setTarget} and {@code SwitchPoint.invalidateAll} fail with + * {@code Unsupported method java.lang.invoke.MethodHandleNatives.setCallSiteTargetNormal}. + * In Groovy's design those two primitives only ever install or invalidate caches — + * the dispatch semantics live entirely in method selection — so under AOT the runtime links + * every site once to its cache-consulting default path ({@code ConstantCallSite}) and carries + * freshness in data instead: + *

+ * The JVM path is untouched: sites link mutable exactly as before, and the stamp is written + * but never read. Coarser than the scoped SwitchPoint invalidation of GROOVY-12191 (any + * invalidation flushes every AOT PIC entry on next hit), which is safe — staleness is + * impossible, over-invalidation just re-selects. + *

+ * The retargeting restriction is particular to GraalVM native image — other ahead-of-time + * or checkpointed runtimes (ART, CRaC, HotSpot AOT caches) retarget call sites normally and + * never need this mode — so auto-detection probes only GraalVM's image-code property. The + * mode itself relies on nothing GraalVM-specific; {@link #FORCE_PROPERTY} is the opt-in for + * any runtime that turns out to share the restriction. + * + * @since 6.0.0 + */ +public final class AotDispatch { + + /** + * Diagnostic knob: forces AOT link mode on a regular JVM so the whole mode can be + * exercised by ordinary tests without a native build. + *

+ * Set it at JVM startup (or before any Groovy code runs) and leave it alone. Because + * {@link #isAotLinkRequested()} is re-evaluated per link and per invalidation, flipping + * the property on mid-run suppresses the real {@link SwitchPoint#invalidateAll} that + * sites already linked in mutable mode depend on — their guards never fire and they + * dispatch stale. (Flipping it off is merely wasteful: sites linked while it was on keep + * consulting the stamp, which keeps advancing, so they stay correct but never regain the + * retargeting fast path.) + */ + public static final String FORCE_PROPERTY = "groovy.indy.aot.link"; + + private static final AtomicLong STAMP = new AtomicLong(); + + private AotDispatch() { + } + + /** + * Whether sites should link in AOT mode. Evaluated per call and never cached in a static: + * under native image this class may be initialized at image build time, where + * {@code org.graalvm.nativeimage.imagecode} reports {@code buildtime} — caching would bake + * the wrong answer into the image heap. Callers are all link-time or invalidation-time + * (cold); per-invocation code reads the site-local flag captured at link time instead. + */ + public static boolean isAotLinkRequested() { + return "runtime".equals(System.getProperty("org.graalvm.nativeimage.imagecode")) + || Boolean.getBoolean(FORCE_PROPERTY); + } + + /** The current global invalidation stamp. */ + public static long stamp() { + return STAMP.get(); + } + + /** + * Invalidates the given switch points, AOT-safely: the global stamp is always advanced + * (so AOT-linked sites observe the change on their next PIC hit), and the actual + * {@link SwitchPoint#invalidateAll} — which native image cannot execute — runs only + * outside AOT mode. All indy invalidation funnels through here. + * + * @param switchPoints the points to invalidate; may be empty + */ + public static void invalidateAll(final SwitchPoint[] switchPoints) { + STAMP.incrementAndGet(); + if (!isAotLinkRequested()) { + SwitchPoint.invalidateAll(switchPoints); + } + } +} diff --git a/src/main/java/org/apache/groovy/runtime/indy/IndyInvalidation.java b/src/main/java/org/apache/groovy/runtime/indy/IndyInvalidation.java index c45793f3e79..e75d3b5979c 100644 --- a/src/main/java/org/apache/groovy/runtime/indy/IndyInvalidation.java +++ b/src/main/java/org/apache/groovy/runtime/indy/IndyInvalidation.java @@ -333,7 +333,8 @@ private static void invalidateBatch(final List batch) { if (batch.isEmpty()) { return; } - SwitchPoint.invalidateAll(batch.toArray(EMPTY_SWITCH_POINTS)); + // AOT-safe: advances the AotDispatch stamp; the real invalidateAll runs only on a JVM + AotDispatch.invalidateAll(batch.toArray(EMPTY_SWITCH_POINTS)); } // ------------------------------------------------------------------------- diff --git a/src/main/java/org/apache/groovy/runtime/indy/SwitchPointInvalidator.java b/src/main/java/org/apache/groovy/runtime/indy/SwitchPointInvalidator.java index 79cb0ddbc0a..4a57a875e34 100644 --- a/src/main/java/org/apache/groovy/runtime/indy/SwitchPointInvalidator.java +++ b/src/main/java/org/apache/groovy/runtime/indy/SwitchPointInvalidator.java @@ -124,7 +124,8 @@ public static void invalidateIfLive(final SwitchPoint sp) { synchronized (SINGLE_INVALIDATE_LOCK) { SINGLE_INVALIDATE_BUF[0] = sp; try { - SwitchPoint.invalidateAll(SINGLE_INVALIDATE_BUF); + // AOT-safe: stamp always advances; real invalidateAll only on a JVM + AotDispatch.invalidateAll(SINGLE_INVALIDATE_BUF); } finally { SINGLE_INVALIDATE_BUF[0] = null; } diff --git a/src/main/java/org/codehaus/groovy/reflection/ClassInfo.java b/src/main/java/org/codehaus/groovy/reflection/ClassInfo.java index ec2042012af..3ec2e09d814 100644 --- a/src/main/java/org/codehaus/groovy/reflection/ClassInfo.java +++ b/src/main/java/org/codehaus/groovy/reflection/ClassInfo.java @@ -26,6 +26,7 @@ import groovy.lang.MetaClassRegistry; import groovy.lang.MetaMethod; import groovy.transform.Internal; +import org.apache.groovy.runtime.indy.AotDispatch; import org.apache.groovy.runtime.indy.IndyInvalidation; import org.apache.groovy.runtime.indy.SwitchPointInvalidator; import org.apache.groovy.util.concurrent.ManagedIdentityConcurrentMap; @@ -239,7 +240,8 @@ public void invalidateIndySwitchPoint() { List batch = new ArrayList<>(2); collectLiveIndySwitchPoints(batch); if (!batch.isEmpty()) { - SwitchPoint.invalidateAll(batch.toArray(new SwitchPoint[0])); + // AOT-safe: stamp always advances; real invalidateAll only on a JVM + AotDispatch.invalidateAll(batch.toArray(new SwitchPoint[0])); } } diff --git a/src/main/java/org/codehaus/groovy/vmplugin/v8/CacheableCallSite.java b/src/main/java/org/codehaus/groovy/vmplugin/v8/CacheableCallSite.java index d412b2b6cb3..b2c17c4e5d9 100644 --- a/src/main/java/org/codehaus/groovy/vmplugin/v8/CacheableCallSite.java +++ b/src/main/java/org/codehaus/groovy/vmplugin/v8/CacheableCallSite.java @@ -58,6 +58,7 @@ public class CacheableCallSite extends MutableCallSite { private volatile SoftReference latestHitMethodHandleWrapperSoftReference = null; private final AtomicLong fallbackCount = new AtomicLong(); private final AtomicLong fallbackRound = new AtomicLong(); + private final boolean aotLinked; private MethodHandle defaultTarget; private MethodHandle fallbackTarget; private final Map> lruCache = @@ -85,6 +86,50 @@ protected boolean removeEldestEntry(Map.Entry eldest) { public CacheableCallSite(MethodType type, MethodHandles.Lookup lookup) { super(type); this.lookup = lookup; + // captured once, at link time (this constructor only runs while linking a site), so + // per-invocation code reads a plain field instead of probing system properties + this.aotLinked = org.apache.groovy.runtime.indy.AotDispatch.isAotLinkRequested(); + } + + /** + * Whether this site was linked in AOT mode (GraalVM native image, or the + * {@code groovy.indy.aot.link} diagnostic knob): the site is wrapped in a + * {@code ConstantCallSite} over the cache-consulting default path, is never retargeted, + * and cache freshness is carried by the {@code AotDispatch} stamp instead of SwitchPoints. + */ + public boolean isAotLinked() { + return aotLinked; + } + + /** + * Fails fast on any retarget attempt in AOT mode. Under a real native image + * {@code setTarget} throws {@code UnsupportedFeatureError} anyway — and worse, execution + * would continue with the stale target if that error were swallowed — so a missed gate is + * a bug on every platform; this surfaces it on the JVM, where tests run with the + * diagnostic knob. + */ + @Override + public void setTarget(final MethodHandle newTarget) { + if (aotLinked) { + throw new IllegalStateException("call site retargeting is disabled in AOT link mode (GROOVY-12234)"); + } + super.setTarget(newTarget); + } + + /** + * Read-only PIC lookup: the cached wrapper for the receiver class, or {@code null} when + * absent or its soft reference has been cleared. Used by the AOT dispatch path, which + * resolves misses itself and must not pay for a value-provider allocation per call. + * + * @param className the receiver cache key + * @return the cached wrapper or {@code null} + */ + public MethodHandleWrapper getIfPresent(String className) { + final SoftReference ref; + synchronized (lruCache) { + ref = lruCache.get(className); + } + return ref == null ? null : ref.get(); } /** diff --git a/src/main/java/org/codehaus/groovy/vmplugin/v8/IndyInterface.java b/src/main/java/org/codehaus/groovy/vmplugin/v8/IndyInterface.java index 99d74334ec5..bdd708cd7f9 100644 --- a/src/main/java/org/codehaus/groovy/vmplugin/v8/IndyInterface.java +++ b/src/main/java/org/codehaus/groovy/vmplugin/v8/IndyInterface.java @@ -20,6 +20,7 @@ import groovy.lang.GroovyRuntimeException; import groovy.lang.GroovySystem; +import org.apache.groovy.runtime.indy.AotDispatch; import org.apache.groovy.runtime.indy.IndyInvalidation; import org.apache.groovy.util.SystemUtil; import org.codehaus.groovy.GroovyBugError; @@ -207,6 +208,12 @@ public int getOrderNumber() { */ static final MethodHandle COLD_REFLECTIVE_INVOKER; + /** + * Handle for {@link #aotDispatch}: the single plain-Java entry an AOT-linked site's + * constant target binds to. + */ + private static final MethodHandle AOT_DISPATCH_METHOD; + static { try { MethodType mt = MethodType.methodType(MethodHandle.class, CacheableCallSite.class, Class.class, String.class, int.class, Boolean.class, Boolean.class, Boolean.class, Object.class, Object[].class); @@ -215,11 +222,47 @@ public int getOrderNumber() { SELECT_METHOD_HANDLE_METHOD = LOOKUP.findStatic(IndyInterface.class, "selectMethodHandle", mt); COLD_REFLECTIVE_INVOKER = LOOKUP.findStatic(IndyInterface.class, "invokeColdReflective", MethodType.methodType(Object.class, ColdReflectiveMethodHandleWrapper.class, Object[].class)); + AOT_DISPATCH_METHOD = LOOKUP.findStatic(IndyInterface.class, "aotDispatch", + MethodType.methodType(Object.class, CacheableCallSite.class, Class.class, String.class, int.class, + Boolean.class, Boolean.class, Boolean.class, Object[].class)); } catch (Exception e) { throw new GroovyBugError(e); } } + /** + * Guards against a GraalVM native-image gap: the runtime invokedynamic linkage invokes a + * bootstrap method without running its declaring class's {@code } first (observed + * on GraalVM CE 25.2.4; on HotSpot the bootstrap's {@code DirectMethodHandle} carries a + * class-initialization barrier). {@link #bootstrap} — the one BSM entry point that reads + * this class's linkage statics — calls this; on an initialized class it is a single null + * check. The other BSM entry points ({@code staticArrayAccess}, {@code packedDispatchers}, + * {@code packedParamTypes}) read no such state and do not need it. + */ + private static void ensureInitialized() { + if (FROM_CACHE_HANDLE_METHOD == null) { + // an ordinary cross-class static read carries the initialization barrier the + // native-image BSM invocation path lacks; reading our own field would not + ClinitBarrier.trigger(); + if (FROM_CACHE_HANDLE_METHOD == null) { + throw new GroovyBugError("IndyInterface linkage handles unavailable: did not run"); + } + } + } + + /** + * A separate class whose read of {@link IndyInterface#LOOKUP} is an ordinary + * {@code getstatic} from foreign code — compiled with the standard ensure-initialized + * barrier that the native-image bootstrap-method invocation path is missing. + */ + private static final class ClinitBarrier { + static void trigger() { + if (IndyInterface.LOOKUP == null) { + throw new GroovyBugError("unreachable: LOOKUP read before initialization"); + } + } + } + static { // MetaClass registry changes invalidate the affected class domain (GROOVY-12191). // Stock MetaClassImpl/EMC → exact class; custom MetaClass kinds → bulk. @@ -294,6 +337,7 @@ static MethodHandle applyMopSwitchPoints(final MethodHandle handle, final Method * @since 2.1.0 */ public static CallSite bootstrap(final MethodHandles.Lookup caller, final String callType, final MethodType type, final String name, final int flags) { + ensureInitialized(); CallType ct = CallType.fromCallSiteName(callType); if (null == ct) throw new GroovyBugError("Unknown call type: " + callType); @@ -314,10 +358,33 @@ public static CallSite bootstrap(final MethodHandles.Lookup caller, final String } // make an adapter for method selection, i.e. get cached method handle (fast path) or fall back MethodHandle mh = makeBootHandle(mc, sender, name, callID, type, safe, thisCall, spreadCall, FROM_CACHE_HANDLE_METHOD); - mc.setTarget(mh); mc.setDefaultTarget(mh); mc.setFallbackTarget(makeFallBack(mc, sender, name, callID, type, safe, thisCall, spreadCall)); + if (mc.isAotLinked()) { + // AOT link mode (GROOVY-12234): native image cannot retarget call sites + // (MethodHandleNatives.setCallSiteTargetNormal is unsupported), so the site links + // once, permanently, to a constant target. The CacheableCallSite is never installed + // as the call site — it serves as the state carrier (PIC, fallback target) the + // dispatcher consults. Retargeting only ever installs caches in this design, so + // semantics are unchanged; freshness moves to the AotDispatch stamp. + // + // The target is deliberately SHALLOW: one bound handle into aotDispatch, which does + // PIC lookup, freshness check, and invocation in ordinary compiled Java, and — for + // the dominant reflective tier — never re-enters the method-handle machinery. + // Measured caveat: entering ANY runtime-created method handle costs ~4.5us under + // native image (a per-entry interpreter cost, independent of chain depth or + // invokeExact), and the invokedynamic instruction's hop into this runtime-linked + // target pays it once per call regardless of the target's shape. The shallow form + // still wins on allocation (no per-call FallbackSupplier/provider lambdas) and + // keeps everything past the entry in compiled code. + MethodHandle aot = MethodHandles.insertArguments(AOT_DISPATCH_METHOD, 0, + mc, sender, name, callID, safe, thisCall, spreadCall); + aot = aot.asCollector(Object[].class, type.parameterCount()).asType(type); + return new ConstantCallSite(aot); + } + mc.setTarget(mh); + return mc; } @@ -438,7 +505,7 @@ private static MethodHandle fromCacheHandle(CacheableCallSite callSite, Class mhw = fallbackSupplier.get(); } - if (mhw.isCanSetTarget() && (callSite.getTarget() != mhw.getTargetMethodHandle())) { + if (!callSite.isAotLinked() && mhw.isCanSetTarget() && (callSite.getTarget() != mhw.getTargetMethodHandle())) { // GROOVY-11935: Set invokedynamic call site target immediately to enable earlier JIT inlining. if (callSite.type().parameterType(0) == Class.class) { var method = mhw.getMethod(); @@ -478,6 +545,58 @@ private static MethodHandle fromCacheHandle(CacheableCallSite callSite, Class return mhw.getCachedMethodHandle(); } + /** + * Samples the invalidation stamp before a selection frame begins, for + * {@link #putSelected}; {@code 0} on a non-AOT site, where the stamp is never consulted. + */ + static long preSelectionStamp(CacheableCallSite callSite) { + return callSite.isAotLinked() ? AotDispatch.stamp() : 0L; + } + + /** + * Stores a freshly selected wrapper in the PIC — unless, on an AOT-linked site, the + * global invalidation stamp moved after {@code preSelectionStamp} was sampled (before + * the selection read any MOP state). The wrapper then reflects a MOP snapshot that may + * predate the change while its construction-time stamp postdates it, so caching it + * could dispatch stale until an unrelated future invalidation; the sentinel is stored + * instead, forcing re-selection on the next hit. (SwitchPoint guards are immune to + * this race because the token is acquired during selection and mutated by the + * invalidation itself; the stamp is compared against a sample, so the sample must be + * taken on the far side of the selection.) Uncacheable selections store the sentinel + * as always. On a non-AOT site this is exactly the historical put. + */ + static void putSelected(CacheableCallSite callSite, String className, MethodHandleWrapper mhw, long preSelectionStamp) { + boolean selectionSpansInvalidation = callSite.isAotLinked() && AotDispatch.stamp() != preSelectionStamp; + callSite.put(className, !selectionSpansInvalidation && mhw.isCanSetTarget() ? mhw : UNCACHEABLE_PIC_SENTINEL); + } + + /** + * The AOT-linked site's dispatch entry: PIC lookup, stamp-based freshness, and invocation, + * all in ordinary compiled Java (see the AOT branch of + * {@link #bootstrap(MethodHandles.Lookup, String, MethodType, String, int)}). The dominant + * tier — the reflective cold wrapper — is invoked directly, not through its bound method + * handle, so steady-state dispatch never enters the native method-handle interpreter. + *

+ * PIC semantics mirror {@code fromCacheHandle}: uncacheable selections store the sentinel + * (forcing re-selection per call, since class-keyed reuse would be wrong for e.g. + * per-instance metaclasses), and a stamp mismatch — any MOP invalidation since the wrapper + * was selected — is a miss. + */ + private static Object aotDispatch(CacheableCallSite callSite, Class sender, String methodName, int callID, + Boolean safeNavigation, Boolean thisCall, Boolean spreadCall, Object[] arguments) throws Throwable { + String receiverClassName = receiverCacheKey(arguments[0]); + MethodHandleWrapper mhw = callSite.getIfPresent(receiverClassName); + if (mhw == null || mhw == UNCACHEABLE_PIC_SENTINEL || mhw.getAotStamp() != AotDispatch.stamp()) { + long preSelectionStamp = preSelectionStamp(callSite); + mhw = fallback(callSite, sender, methodName, callID, safeNavigation, thisCall, spreadCall, 1, arguments); + putSelected(callSite, receiverClassName, mhw, preSelectionStamp); + } + if (mhw instanceof ColdReflectiveMethodHandleWrapper) { + return invokeColdReflective((ColdReflectiveMethodHandleWrapper) mhw, arguments); + } + return mhw.getCachedMethodHandle().invokeExact(arguments); + } + /** * Cold-tier dispatch for the {@code groovy.indy.cold.reflection} spike. * Re-validates the cached selection with plain-Java checks and invokes the @@ -497,8 +616,14 @@ private static MethodHandle fromCacheHandle(CacheableCallSite callSite, Class * reflective path on the same miss. */ private static Object invokeColdReflective(ColdReflectiveMethodHandleWrapper cold, Object[] arguments) throws Throwable { - if (cold.isValidFor(arguments)) { - if (cold.incrementReflectiveHits() > INDY_OPTIMIZE_THRESHOLD) { + // AOT freshness: classValidity.hasBeenInvalidated() can never report true under native + // image, so the stamp carries staleness; a mismatch takes the re-selection path below + boolean aotStale = cold.callSite.isAotLinked() && cold.getAotStamp() != AotDispatch.stamp(); + if (!aotStale && cold.isValidFor(arguments)) { + // In AOT mode the reflective tier IS the steady state: method-handle chains run in + // the native MH interpreter (microseconds/call, no JIT to fold them) while + // reflective dispatch uses AOT-compiled invocation stubs — so never promote. + if (!cold.callSite.isAotLinked() && cold.incrementReflectiveHits() > INDY_OPTIMIZE_THRESHOLD) { // no longer cold: build the full guarded chain and replace the // PIC entry, so even sites the consecutive-hit promotion never // catches (e.g. polymorphic receivers) leave the reflective @@ -520,9 +645,27 @@ private static Object invokeColdReflective(ColdReflectiveMethodHandleWrapper col throw ScriptBytecodeAdapter.unwrap(gre); } } + if (aotStale) { + // A stamp mismatch means the MOP changed, not that this selection shape is + // problematic, so re-select with the cold tier still allowed: the AOT steady + // state must remain the reflective wrapper (a full chain would run in the + // native method-handle interpreter until the next stamp bump). The fresh + // wrapper captures the current stamp, so this cannot recurse on the same + // cause; the GROOVY-12191 recursion below stems from validity failures, + // which under AOT never come from SwitchPoints. + long preSelectionStamp = preSelectionStamp(cold.callSite); + MethodHandleWrapper fresh = fallback(cold.callSite, cold.sender, cold.methodName, cold.callID, + cold.safeNavigation, cold.thisCall, cold.spreadCall, 1, arguments); + putSelected(cold.callSite, receiverCacheKey(arguments[0]), fresh, preSelectionStamp); + if (fresh instanceof ColdReflectiveMethodHandleWrapper) { + return invokeColdReflective((ColdReflectiveMethodHandleWrapper) fresh, arguments); + } + return fresh.getCachedMethodHandle().invokeExact(arguments); + } // Re-select without the cold tier so an always-invalid SwitchPoint (or any // permanent cold miss after class-domain failover) cannot recurse through // tryBuild → invokeColdReflective (GROOVY-12191). + long preSelectionStamp = preSelectionStamp(cold.callSite); MethodHandleWrapper full = fallback(cold.callSite, cold.sender, cold.methodName, cold.callID, cold.safeNavigation, cold.thisCall, cold.spreadCall, 1, arguments, false); // PIC write policy — same as fromCacheHandle / selectMethodHandle: @@ -541,8 +684,7 @@ private static Object invokeColdReflective(ColdReflectiveMethodHandleWrapper col // // UNCACHEABLE_PIC_SENTINEL is therefore the only safe PIC value when // the re-selected wrapper must not become a class-keyed cache entry. - cold.callSite.put(receiverCacheKey(arguments[0]), - full.isCanSetTarget() ? full : UNCACHEABLE_PIC_SENTINEL); + putSelected(cold.callSite, receiverCacheKey(arguments[0]), full, preSelectionStamp); return full.getCachedMethodHandle().invokeExact(arguments); } @@ -560,23 +702,28 @@ public static Object selectMethod(CacheableCallSite callSite, Class sender, S * Core method for indy method selection using runtime types. */ private static MethodHandle selectMethodHandle(CacheableCallSite callSite, Class sender, String methodName, int callID, Boolean safeNavigation, Boolean thisCall, Boolean spreadCall, Object dummyReceiver, Object[] arguments) throws Throwable { + long preSelectionStamp = preSelectionStamp(callSite); MethodHandleWrapper mhw = fallback(callSite, sender, methodName, callID, safeNavigation, thisCall, spreadCall, dummyReceiver, arguments); MethodHandle defaultTarget = callSite.getDefaultTarget(); long fallbackCount = callSite.incrementFallbackCount(); - if ((fallbackCount > INDY_FALLBACK_THRESHOLD) && (callSite.getTarget() != defaultTarget)) { + if (!callSite.isAotLinked() + && (fallbackCount > INDY_FALLBACK_THRESHOLD) && (callSite.getTarget() != defaultTarget)) { callSite.setTarget(defaultTarget); if (LOG_ENABLED) LOG.info("call site target reset to default, preparing outside invocation"); callSite.resetFallbackCount(); } - if (callSite.getTarget() == defaultTarget) { + // in AOT mode the effective target is always the default path (the ConstantCallSite + // wraps it), so the PIC write-back below must run; getTarget() would report the + // never-installed placeholder + if (callSite.isAotLinked() || callSite.getTarget() == defaultTarget) { // correct the stale methodHandle in the inline cache of callsite // it is important but impacts the performance somehow when cache misses frequently Object receiver = arguments[0]; // Avoid PIC pollution: don't write back uncached wrappers, e.g. for instance-level metaClass dispatches. - callSite.put(receiverCacheKey(receiver), mhw.isCanSetTarget() ? mhw : UNCACHEABLE_PIC_SENTINEL); + putSelected(callSite, receiverCacheKey(receiver), mhw, preSelectionStamp); } return mhw.getCachedMethodHandle(); diff --git a/src/main/java/org/codehaus/groovy/vmplugin/v8/MethodHandleWrapper.java b/src/main/java/org/codehaus/groovy/vmplugin/v8/MethodHandleWrapper.java index 33ff56fb1f4..cb70e86a81e 100644 --- a/src/main/java/org/codehaus/groovy/vmplugin/v8/MethodHandleWrapper.java +++ b/src/main/java/org/codehaus/groovy/vmplugin/v8/MethodHandleWrapper.java @@ -34,6 +34,16 @@ class MethodHandleWrapper { private final MetaMethod method; private final boolean canSetTarget; private final AtomicLong latestHitCount = new AtomicLong(0); + /** + * The global {@code AotDispatch} invalidation stamp at creation. Read only for sites + * linked in AOT mode: a mismatch on a PIC hit means the MOP changed since this wrapper + * was selected and it must be treated as a miss (the AOT replacement for the SwitchPoint + * guards, which cannot fire under native image). Construction runs at the end of + * selection, so this stamp only covers invalidations arriving after caching; one landing + * while selection itself runs is caught at the PIC write instead + * (see {@code IndyInterface#putSelected}). + */ + private final long aotStamp = org.apache.groovy.runtime.indy.AotDispatch.stamp(); /** * Creates a wrapper for the cached and relink targets of a meta method. @@ -77,6 +87,16 @@ public MetaMethod getMethod() { return method; } + /** + * Returns the global {@code AotDispatch} invalidation stamp captured at creation + * (see the {@code aotStamp} field for how AOT-linked sites use it). + * + * @return the creation-time stamp + */ + long getAotStamp() { + return aotStamp; + } + /** * Indicates whether this wrapper may be installed as the call-site target. * diff --git a/src/test/groovy/org/apache/groovy/runtime/indy/AotLinkModeTest.groovy b/src/test/groovy/org/apache/groovy/runtime/indy/AotLinkModeTest.groovy new file mode 100644 index 00000000000..a254f5a3eef --- /dev/null +++ b/src/test/groovy/org/apache/groovy/runtime/indy/AotLinkModeTest.groovy @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.groovy.runtime.indy + +import org.codehaus.groovy.vmplugin.v8.CacheableCallSite +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.parallel.ResourceLock +import org.junit.jupiter.api.parallel.Resources + +import java.lang.invoke.MethodHandles +import java.lang.invoke.MethodType +import java.lang.invoke.SwitchPoint + +import static org.junit.jupiter.api.Assertions.assertThrows + +/** + * AOT link mode (GROOVY-12234) exercised on a regular JVM through the + * {@link AotDispatch#FORCE_PROPERTY} diagnostic knob: sites link once to a constant + * cache-consulting target, are never retargeted, and carry cache freshness in the global + * invalidation stamp instead of SwitchPoints. + *

+ * The property is captured per site at link time, so every test evaluates freshly compiled + * scripts inside the property window; their call sites all link in AOT mode. Meta classes are + * only ever mutated on script-local classes — while the window is open, real SwitchPoint + * invalidation is suppressed, so mutating a shared class's meta class could leave sites + * outside the window stale (the documented mid-flip hazard of {@code FORCE_PROPERTY}). + */ +@ResourceLock(Resources.SYSTEM_PROPERTIES) +final class AotLinkModeTest { + + private static T withAotLink(Closure work) { + String previous = System.getProperty(AotDispatch.FORCE_PROPERTY) + System.setProperty(AotDispatch.FORCE_PROPERTY, 'true') + try { + work.call() + } finally { + if (previous != null) { + System.setProperty(AotDispatch.FORCE_PROPERTY, previous) + } else { + System.clearProperty(AotDispatch.FORCE_PROPERTY) + } + } + } + + private static Object evaluateAotLinked(String script) { + withAotLink { + new GroovyShell().evaluate(script) + } + } + + /** + * Every dispatch shape links and runs on AOT-linked sites. The hot loop runs past + * {@code groovy.indy.fallback.threshold} and {@code groovy.indy.optimize.threshold} + * (both default 1000): any missed retarget gate on the promotion or reset paths would + * surface as the fail-fast {@code IllegalStateException} from + * {@code CacheableCallSite.setTarget}. + */ + @Test + void 'dispatch gauntlet runs correctly on AOT-linked sites'() { + def result = evaluateAotLinked ''' + class Calc { + int base = 40 + int add(int x) { base + x } + static String greet(String who) { "hi $who" } + } + def out = [] + def c = new Calc() + out << c.add(2) // instance method + out << Calc.greet('aot') // static method + out << c.base // property get + c.base = 1 + out << c.base // property set + out << new Calc().add(41) // constructor + out << [1, 2, 3].collect { it * 2 } // GDK + closure + def sum = 0 + for (i in 1..2500) { sum += c.add(i) } // hot: past both thresholds + out << sum + out + ''' + assert result == [42, 'hi aot', 40, 1, 81, [2, 4, 6], 3128750] + } + + /** Polymorphic receivers churn one site's PIC without ever needing a retarget. */ + @Test + void 'polymorphic dispatch runs correctly on one AOT-linked site'() { + def result = evaluateAotLinked ''' + class A { String id() { 'a' } } + class B { String id() { 'b' } } + class C { String id() { 'c' } } + def call = { it.id() } // single call site + def receivers = [new A(), new B(), new C()] + (1..300).collect { call(receivers[it % 3]) }.unique().sort() + ''' + assert result == ['a', 'b', 'c'] + } + + /** + * The stamp is the AOT replacement for SwitchPoint guards: a meta class change after a + * site has linked and cached its selection must be observed on the next call through + * that same site. With the property forced, real SwitchPoint invalidation is skipped, + * so this passing proves the stamp flush alone carries the change. Each call dispatches + * on a fresh receiver: an instance that dispatched before the change keeps the meta + * class captured in its instance field on either path (plain-JVM parity, verified), + * which would test instance-capture semantics rather than the site's cache. + */ + @Test + void 'meta class change is observed through the stamp on an already-hot site'() { + def result = evaluateAotLinked ''' + class Subject { String speak() { 'original' } } + def call = { -> new Subject().speak() } // the one site under test + def first = (1..50).collect { call() }.unique() + Subject.metaClass.speak = { -> 'intercepted' } + [first, call()] + ''' + assert result == [['original'], 'intercepted'] + } + + /** + * A per-instance meta class is not class-keyed-cacheable: the sentinel must force + * re-selection on every later hit, keeping plain and per-instance receivers correct + * through the same AOT-linked site in any order. + */ + @Test + void 'per-instance meta class re-selects through the sentinel'() { + def result = evaluateAotLinked ''' + class Duo { String name() { 'plain' } } + def call = { Duo d -> d.name() } // the one site under test + def a = new Duo() + def b = new Duo() + call(a) // cache the plain selection + b.metaClass.name = { -> 'special' } + [call(a), call(b), call(a), call(b)] + ''' + assert result == ['plain', 'special', 'plain', 'special'] + } + + @Test + void 'setTarget fails fast on an AOT-linked site and works on a normal one'() { + withAotLink { + def aotSite = new CacheableCallSite(MethodType.methodType(Object, Object[]), MethodHandles.lookup()) + def e = assertThrows(IllegalStateException) { + aotSite.setTarget(MethodHandles.empty(aotSite.type())) + } + assert e.message.contains('AOT link mode') + } + def normalSite = new CacheableCallSite(MethodType.methodType(Object, Object[]), MethodHandles.lookup()) + normalSite.setTarget(MethodHandles.empty(normalSite.type())) // no throw + } + + @Test + void 'invalidation always advances the stamp and suppresses real invalidation only in AOT mode'() { + def suppressed = new SwitchPoint() + withAotLink { + long before = AotDispatch.stamp() + AotDispatch.invalidateAll([suppressed] as SwitchPoint[]) + assert AotDispatch.stamp() == before + 1 + assert !suppressed.hasBeenInvalidated() + } + def invalidated = new SwitchPoint() + long before = AotDispatch.stamp() + AotDispatch.invalidateAll([invalidated] as SwitchPoint[]) + assert AotDispatch.stamp() == before + 1 + assert invalidated.hasBeenInvalidated() + } +} diff --git a/src/test/groovy/org/codehaus/groovy/vmplugin/v8/AotPutSelectedTest.groovy b/src/test/groovy/org/codehaus/groovy/vmplugin/v8/AotPutSelectedTest.groovy new file mode 100644 index 00000000000..de152b782f3 --- /dev/null +++ b/src/test/groovy/org/codehaus/groovy/vmplugin/v8/AotPutSelectedTest.groovy @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.codehaus.groovy.vmplugin.v8 + +import org.apache.groovy.runtime.indy.AotDispatch +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.parallel.ResourceLock +import org.junit.jupiter.api.parallel.Resources + +import java.lang.invoke.MethodHandles +import java.lang.invoke.MethodType +import java.lang.invoke.SwitchPoint + +/** + * The pre-selection stamp guard on AOT PIC writes ({@code IndyInterface.putSelected}), + * deterministically: a wrapper selected while an invalidation landed reflects a MOP snapshot + * that may predate the change, yet its construction-time stamp postdates it — caching it + * would dispatch stale until an unrelated future invalidation, since under AOT no + * SwitchPoint ever fires. The guard caches the sentinel instead, forcing re-selection on + * the next hit. The race window is selection-start to PIC-write; these tests recreate it + * by bumping the stamp between sampling and the write, no actual thread race required. + */ +@ResourceLock(Resources.SYSTEM_PROPERTIES) +final class AotPutSelectedTest { + + private static CacheableCallSite siteLinkedWithAot(boolean aot) { + String previous = System.getProperty(AotDispatch.FORCE_PROPERTY) + if (aot) System.setProperty(AotDispatch.FORCE_PROPERTY, 'true') + else System.clearProperty(AotDispatch.FORCE_PROPERTY) + try { + new CacheableCallSite(MethodType.methodType(Object, Object[]), MethodHandles.lookup()) + } finally { + if (previous != null) { + System.setProperty(AotDispatch.FORCE_PROPERTY, previous) + } else { + System.clearProperty(AotDispatch.FORCE_PROPERTY) + } + } + } + + private static MethodHandleWrapper cacheableWrapper() { + def mh = MethodHandles.empty(MethodType.methodType(Object, Object[])) + new MethodHandleWrapper(mh, mh, null, true) + } + + private static void bumpStamp() { + AotDispatch.invalidateAll(new SwitchPoint[0]) + } + + @Test + void 'a selection no invalidation raced is cached on an AOT site'() { + def site = siteLinkedWithAot(true) + def wrapper = cacheableWrapper() + long pre = IndyInterface.preSelectionStamp(site) + IndyInterface.putSelected(site, 'K', wrapper, pre) + assert site.getIfPresent('K').is(wrapper) + } + + @Test + void 'a selection an invalidation raced caches the sentinel, never the possibly-stale wrapper'() { + def site = siteLinkedWithAot(true) + def wrapper = cacheableWrapper() // construction-time stamp postdates the bump below on a real race + long pre = IndyInterface.preSelectionStamp(site) + bumpStamp() // the invalidation landing mid-selection, made deterministic + IndyInterface.putSelected(site, 'K', wrapper, pre) + assert site.getIfPresent('K').is(MethodHandleWrapper.uncacheablePicSentinel) + } + + @Test + void 'an uncacheable selection stores the sentinel regardless of the stamp'() { + def site = siteLinkedWithAot(true) + def mh = MethodHandles.empty(MethodType.methodType(Object, Object[])) + def uncacheable = new MethodHandleWrapper(mh, mh, null, false) + long pre = IndyInterface.preSelectionStamp(site) + IndyInterface.putSelected(site, 'K', uncacheable, pre) + assert site.getIfPresent('K').is(MethodHandleWrapper.uncacheablePicSentinel) + } + + @Test + void 'on a non-AOT site the stamp is never consulted and the historical put is preserved'() { + def site = siteLinkedWithAot(false) + def wrapper = cacheableWrapper() + long pre = IndyInterface.preSelectionStamp(site) + assert pre == 0L + bumpStamp() // irrelevant off the AOT path: SwitchPoint guards carry freshness there + IndyInterface.putSelected(site, 'K', wrapper, pre) + assert site.getIfPresent('K').is(wrapper) + } +}