Add test support to record async events, with Junit5 caveat

This commit modifies the way the `@RecordApplicationEvents` annotation
works in tests, allowing for capture of events from threads other than
the main test thread (async events) and for the assertion of captured
event from a separate thread (e.g. when using `Awaitility`).

This is done by switching the `ApplicationEventsHolder` to use an
`InheritedThreadLocal`.

There is a mutual exclusion between support of asynchronous events vs
support of JUnit5 parallel tests with the `@TestInstance(PER_CLASS)`
mode. As a result, we favor the former and now `SpringExtension` will
invalidate a test class that is annotated (or meta-annotated, or
enclosed-annotated) with `@RecordApplicationEvents` AND
`@TestInstance(PER_CLASS)` AND `@Execution(CONCURRENT)`.

See gh-29827
Closes gh-30020

spring-test/spring-test.gradle

@@ -76,6 +76,7 @@ dependencies {
 	}
 	testImplementation("io.projectreactor.netty:reactor-netty-http")
 	testImplementation("de.bechte.junit:junit-hierarchicalcontextrunner")
+	testImplementation("org.awaitility:awaitility")
 	testRuntimeOnly("org.junit.vintage:junit-vintage-engine") {
 		exclude group: "junit", module: "junit"
 	}

spring-test/src/main/java/org/springframework/test/context/event/ApplicationEventsHolder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2021 the original author or authors.
+ * Copyright 2002-2023 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ import org.springframework.util.Assert;
  *
  * @author Sam Brannen
  * @author Oliver Drotbohm
+ * @author Simon Baslé
  * @since 5.3.3
  * @see ApplicationEvents
  * @see RecordApplicationEvents
@@ -44,7 +45,7 @@ import org.springframework.util.Assert;
  */
 public abstract class ApplicationEventsHolder {
 
-	private static final ThreadLocal<DefaultApplicationEvents> applicationEvents = new ThreadLocal<>();
+	private static final ThreadLocal<DefaultApplicationEvents> applicationEvents = new InheritableThreadLocal<>();
 
 
 	private ApplicationEventsHolder() {

spring-test/src/main/java/org/springframework/test/context/event/DefaultApplicationEvents.java

@@ -16,8 +16,8 @@
 
 package org.springframework.test.context.event;
 
-import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.stream.Stream;
 
 import org.springframework.context.ApplicationEvent;
@@ -32,7 +32,7 @@ import org.springframework.context.PayloadApplicationEvent;
  */
 class DefaultApplicationEvents implements ApplicationEvents {
 
-	private final List<ApplicationEvent> events = new ArrayList<>();
+	private final List<ApplicationEvent> events = new CopyOnWriteArrayList<>();
 
 
 	void addEvent(ApplicationEvent event) {

spring-test/src/main/java/org/springframework/test/context/junit/jupiter/SpringExtension.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2022 the original author or authors.
+ * Copyright 2002-2023 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,6 +29,7 @@ import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.extension.AfterAllCallback;
 import org.junit.jupiter.api.extension.AfterEachCallback;
 import org.junit.jupiter.api.extension.AfterTestExecutionCallback;
@@ -41,6 +42,8 @@ import org.junit.jupiter.api.extension.ExtensionContext.Store;
 import org.junit.jupiter.api.extension.ParameterContext;
 import org.junit.jupiter.api.extension.ParameterResolver;
 import org.junit.jupiter.api.extension.TestInstancePostProcessor;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.platform.commons.annotation.Testable;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -51,8 +54,10 @@ import org.springframework.core.annotation.MergedAnnotations.SearchStrategy;
 import org.springframework.core.annotation.RepeatableContainers;
 import org.springframework.lang.Nullable;
 import org.springframework.test.context.TestConstructor;
+import org.springframework.test.context.TestContextAnnotationUtils;
 import org.springframework.test.context.TestContextManager;
 import org.springframework.test.context.event.ApplicationEvents;
+import org.springframework.test.context.event.RecordApplicationEvents;
 import org.springframework.test.context.support.PropertyProvider;
 import org.springframework.test.context.support.TestConstructorUtils;
 import org.springframework.util.Assert;
@@ -68,6 +73,7 @@ import org.springframework.util.ReflectionUtils.MethodFilter;
  * {@code @SpringJUnitWebConfig}.
  *
  * @author Sam Brannen
+ * @author Simon Baslé
  * @since 5.0
  * @see org.springframework.test.context.junit.jupiter.EnabledIf
  * @see org.springframework.test.context.junit.jupiter.DisabledIf
@@ -94,6 +100,13 @@ public class SpringExtension implements BeforeAllCallback, AfterAllCallback, Tes
 
 	private static final String NO_AUTOWIRED_VIOLATIONS_DETECTED = "NO AUTOWIRED VIOLATIONS DETECTED";
 
+	/**
+	 * {@link Namespace} in which {@code @RecordApplicationEvents} validation error messages
+	 * are stored, keyed by test class.
+	 */
+	private static final Namespace RECORD_APPLICATION_EVENTS_VALIDATION_NAMESPACE =
+			Namespace.create(SpringExtension.class.getName() + "#recordApplicationEvents.validation");
+
 	// Note that @Test, @TestFactory, @TestTemplate, @RepeatedTest, and @ParameterizedTest
 	// are all meta-annotated with @Testable.
 	private static final List<Class<? extends Annotation>> JUPITER_ANNOTATION_TYPES =
@@ -135,9 +148,51 @@ public class SpringExtension implements BeforeAllCallback, AfterAllCallback, Tes
 	@Override
 	public void postProcessTestInstance(Object testInstance, ExtensionContext context) throws Exception {
 		validateAutowiredConfig(context);
+		validateRecordApplicationEventsConfig(context);
 		getTestContextManager(context).prepareTestInstance(testInstance);
 	}
 
+	/**
+	 * Validate that test class or its enclosing class doesn't attempt to record
+	 * application events in a parallel mode that makes it un-deterministic
+	 * ({@code @TestInstance(PER_CLASS)} and {@code @Execution(CONCURRENT)}
+	 * combination).
+	 * @since 6.1.0
+	 */
+	private void validateRecordApplicationEventsConfig(ExtensionContext context) {
+		// We save the result in the ExtensionContext.Store so that we don't
+		// re-validate all methods for the same test class multiple times.
+		Store store = context.getStore(RECORD_APPLICATION_EVENTS_VALIDATION_NAMESPACE);
+
+		String errorMessage = store.getOrComputeIfAbsent(context.getRequiredTestClass(), testClass -> {
+			boolean record = TestContextAnnotationUtils.hasAnnotation(testClass, RecordApplicationEvents.class);
+			if (!record) {
+				return NO_AUTOWIRED_VIOLATIONS_DETECTED;
+			}
+			final TestInstance testInstance = TestContextAnnotationUtils.findMergedAnnotation(testClass, TestInstance.class);
+
+			if (testInstance == null || testInstance.value() != TestInstance.Lifecycle.PER_CLASS) {
+				return NO_AUTOWIRED_VIOLATIONS_DETECTED;
+			}
+
+			final Execution execution = TestContextAnnotationUtils.findMergedAnnotation(testClass, Execution.class);
+
+			if (execution == null || execution.value() != ExecutionMode.CONCURRENT) {
+				return NO_AUTOWIRED_VIOLATIONS_DETECTED;
+			}
+
+			return "Test classes or inner classes that @RecordApplicationEvents must not be run in parallel "
+					+ "with the @TestInstance(Lifecycle.PER_CLASS) configuration. Use either @Execution(SAME_THREAD), "
+					+ "@TestInstance(PER_METHOD) or disable parallel execution altogether. Note that when recording "
+					+ "events in parallel, one might see events published by other tests as the application context "
+					+ "can be common.";
+		}, String.class);
+
+		if (errorMessage != NO_AUTOWIRED_VIOLATIONS_DETECTED) {
+			throw new IllegalStateException(errorMessage);
+		}
+	}
+
 	/**
 	 * Validate that test methods and test lifecycle methods in the supplied
 	 * test class are not annotated with {@link Autowired @Autowired}.

spring-test/src/test/java/org/springframework/test/context/junit/jupiter/event/JUnitJupiterApplicationEventsIntegrationTests.java

@@ -18,6 +18,9 @@ package org.springframework.test.context.junit.jupiter.event;
 
 import java.util.stream.Stream;
 
+import org.assertj.core.api.InstanceOfAssertFactories;
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Nested;
@@ -237,6 +240,38 @@ class JUnitJupiterApplicationEventsIntegrationTests {
 		}
 	}
 
+	@Nested
+	@TestInstance(PER_CLASS)
+	class AsyncEventTests {
+
+		@Autowired
+		ApplicationEvents applicationEvents;
+
+		@Test
+		void asyncPublication() throws InterruptedException {
+			Thread t = new Thread(() -> context.publishEvent(new CustomEvent("async")));
+			t.start();
+			t.join();
+
+			assertThat(this.applicationEvents.stream(CustomEvent.class))
+					.singleElement()
+					.extracting(CustomEvent::getMessage, InstanceOfAssertFactories.STRING)
+					.isEqualTo("async");
+		}
+
+		@Test
+		void asyncConsumption() {
+			context.publishEvent(new CustomEvent("sync"));
+
+			Awaitility.await().atMost(Durations.ONE_SECOND)
+					.untilAsserted(() -> assertThat(assertThat(this.applicationEvents.stream(CustomEvent.class))
+							.singleElement()
+							.extracting(CustomEvent::getMessage, InstanceOfAssertFactories.STRING)
+							.isEqualTo("sync")));
+		}
+
+	}
+
 
 	private static void assertEventTypes(ApplicationEvents applicationEvents, String... types) {
 		assertThat(applicationEvents.stream().map(event -> event.getClass().getSimpleName()))

spring-test/src/test/java/org/springframework/test/context/junit/jupiter/event/ParallelApplicationEventsIntegrationTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2021 the original author or authors.
+ * Copyright 2002-2023 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,9 +18,15 @@ package org.springframework.test.context.junit.jupiter.event;
 
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.assertj.core.api.InstanceOfAssertFactories;
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
@@ -28,15 +34,19 @@ import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.ExecutionMode;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.platform.engine.TestExecutionResult;
+import org.junit.platform.testkit.engine.EngineExecutionResults;
 import org.junit.platform.testkit.engine.EngineTestKit;
+import org.junit.platform.testkit.engine.Events;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
+import org.springframework.context.PayloadApplicationEvent;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.test.context.event.ApplicationEvents;
+import org.springframework.test.context.event.ApplicationEventsHolder;
 import org.springframework.test.context.event.RecordApplicationEvents;
+import org.springframework.test.context.event.TestContextEvent;
 import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -47,23 +57,52 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass
  * in conjunction with JUnit Jupiter.
  *
  * @author Sam Brannen
+ * @author Simon Baslé
  * @since 5.3.3
  */
 class ParallelApplicationEventsIntegrationTests {
 
 	private static final Set<String> payloads = ConcurrentHashMap.newKeySet();
 
+	@Test
+	void rejectTestsInParallelWithInstancePerClassAndRecordApplicationEvents() {
+		Class<?> testClass = TestInstancePerClassTestCase.class;
 
-	@ParameterizedTest
-	@ValueSource(classes = {TestInstancePerMethodTestCase.class, TestInstancePerClassTestCase.class})
-	void executeTestsInParallel(Class<?> testClass) {
-		EngineTestKit.engine("junit-jupiter")//
+		final EngineExecutionResults results = EngineTestKit.engine("junit-jupiter")//
+				.selectors(selectClass(testClass))//
+				.configurationParameter("junit.jupiter.execution.parallel.enabled", "true")//
+				.configurationParameter("junit.jupiter.execution.parallel.config.dynamic.factor", "10")//
+				.execute();
+
+		//extract the messages from failed TextExecutionResults
+		assertThat(results.containerEvents().failed()//
+				.stream().map(e -> e.getRequiredPayload(TestExecutionResult.class)//
+				.getThrowable().get().getMessage()))//
+				.singleElement(InstanceOfAssertFactories.STRING)
+				.isEqualToIgnoringNewLines("""
+		Test classes or inner classes that @RecordApplicationEvents\s
+		must not be run in parallel with the @TestInstance(Lifecycle.PER_CLASS) configuration.\s
+		Use either @Execution(SAME_THREAD), @TestInstance(PER_METHOD) or disable parallel\s
+		execution altogether. Note that when recording events in parallel, one might see events\s
+		published by other tests as the application context can be common.
+		""");
+	}
+
+	@Test
+	void executeTestsInParallelInstancePerMethod() {
+		Class<?> testClass = TestInstancePerMethodTestCase.class;
+		Events testEvents = EngineTestKit.engine("junit-jupiter")//
 				.selectors(selectClass(testClass))//
 				.configurationParameter("junit.jupiter.execution.parallel.enabled", "true")//
 				.configurationParameter("junit.jupiter.execution.parallel.config.dynamic.factor", "10")//
 				.execute()//
-				.testEvents()//
-				.assertStatistics(stats -> stats.started(10).succeeded(10).failed(0));
+				.testEvents();
+		//list failed events in case of test errors to get a sense of which tests failed
+		Events failedTests = testEvents.failed();
+		if (failedTests.count() > 0) {
+			failedTests.debug();
+		}
+		testEvents.assertStatistics(stats -> stats.started(13).succeeded(13).failed(0));
 
 		Set<String> testNames = payloads.stream()//
 				.map(payload -> payload.substring(0, payload.indexOf("-")))//
@@ -162,6 +201,39 @@ class ParallelApplicationEventsIntegrationTests {
 			assertTestExpectations(events, testInfo);
 		}
 
+		@Test
+		void compareToApplicationEventsHolder(ApplicationEvents applicationEvents) {
+			ApplicationEvents fromThreadHolder = ApplicationEventsHolder.getRequiredApplicationEvents();
+			assertThat(fromThreadHolder.stream())
+					.hasSameElementsAs(this.events.stream().toList())
+					.hasSameElementsAs(applicationEvents.stream().toList());
+		}
+
+		@Test
+		void asyncPublication(ApplicationEvents events) throws InterruptedException {
+			final ExecutorService executorService = Executors.newSingleThreadExecutor();
+			executorService.execute(() -> this.context.publishEvent("asyncPublication"));
+			executorService.shutdown();
+			executorService.awaitTermination(10, TimeUnit.SECONDS);
+
+			assertThat(events.stream().filter(e -> !(e instanceof TestContextEvent))
+					.map(e -> (e instanceof PayloadApplicationEvent<?> pae ? pae.getPayload().toString() : e.toString())))
+					.containsExactly("asyncPublication");
+		}
+
+		@Test
+		void asyncConsumption() {
+			this.context.publishEvent("asyncConsumption");
+
+			Awaitility.await().atMost(Durations.ONE_SECOND).untilAsserted(() ->//
+					assertThat(ApplicationEventsHolder//
+							.getRequiredApplicationEvents()//
+							.stream()//
+							.filter(e -> !(e instanceof TestContextEvent))//
+							.map(e -> (e instanceof PayloadApplicationEvent<?> pae ? pae.getPayload().toString() : e.toString()))//
+					).containsExactly("asyncConsumption"));
+		}
+
 		private void assertTestExpectations(ApplicationEvents events, TestInfo testInfo) {
 			String testName = testInfo.getTestMethod().get().getName();
 			String threadName = Thread.currentThread().getName();

spring-test/src/test/java/org/springframework/test/context/junit4/event/JUnit4ApplicationEventsAsyncIntegrationTests.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2002-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.test.context.junit4.event;
+
+import org.assertj.core.api.InstanceOfAssertFactories;
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.event.ApplicationEvents;
+import org.springframework.test.context.event.RecordApplicationEvents;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.context.junit4.event.JUnit4ApplicationEventsIntegrationTests.CustomEvent;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration tests for {@link ApplicationEvents} that record async events
+ * or assert the events from a separate thread, in conjunction with JUnit 4.
+ *
+ * @author Simon Baslé
+ * @since 6.1.0
+ */
+@RunWith(SpringRunner.class)
+@RecordApplicationEvents
+public class JUnit4ApplicationEventsAsyncIntegrationTests {
+
+	@Rule
+	public final TestName testName = new TestName();
+
+	@Autowired
+	ApplicationContext context;
+
+	@Autowired
+	ApplicationEvents applicationEvents;
+
+	@Test
+	public void asyncPublication() throws InterruptedException {
+		Thread t = new Thread(() -> context.publishEvent(new CustomEvent("async")));
+		t.start();
+		t.join();
+
+		assertThat(this.applicationEvents.stream(CustomEvent.class))
+				.singleElement()
+				.extracting(CustomEvent::getMessage, InstanceOfAssertFactories.STRING)
+				.isEqualTo("async");
+	}
+
+	@Test
+	public void asyncConsumption() {
+		context.publishEvent(new CustomEvent("sync"));
+
+		Awaitility.await().atMost(Durations.ONE_SECOND)
+				.untilAsserted(() -> assertThat(assertThat(this.applicationEvents.stream(CustomEvent.class))
+						.singleElement()
+						.extracting(CustomEvent::getMessage, InstanceOfAssertFactories.STRING)
+						.isEqualTo("sync")));
+	}
+
+
+	@Configuration
+	static class Config {
+	}
+}

spring-test/src/test/java/org/springframework/test/context/testng/event/TestNGApplicationEventsAsyncIntegrationTests.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2002-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.test.context.testng.event;
+
+import org.assertj.core.api.InstanceOfAssertFactories;
+import org.awaitility.Awaitility;
+import org.awaitility.Durations;
+import org.testng.annotations.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.event.ApplicationEvents;
+import org.springframework.test.context.event.RecordApplicationEvents;
+import org.springframework.test.context.testng.AbstractTestNGSpringContextTests;
+import org.springframework.test.context.testng.event.TestNGApplicationEventsIntegrationTests.CustomEvent;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration tests for {@link ApplicationEvents} that record async events
+ * or assert the events from a separate thread, in conjunction with TestNG.
+ *
+ * @author Simon Baslé
+ * @since 6.1.0
+ */
+@RecordApplicationEvents
+class TestNGApplicationEventsAsyncIntegrationTests extends AbstractTestNGSpringContextTests {
+
+	@Autowired
+	ApplicationContext context;
+
+	@Autowired
+	ApplicationEvents applicationEvents;
+
+
+	@Test
+	public void asyncPublication() throws InterruptedException {
+		Thread t = new Thread(() -> context.publishEvent(new CustomEvent("asyncPublication")));
+		t.start();
+		t.join();
+
+		assertThat(this.applicationEvents.stream(CustomEvent.class))
+				.singleElement()
+				.extracting(CustomEvent::getMessage, InstanceOfAssertFactories.STRING)
+				.isEqualTo("asyncPublication");
+	}
+
+	@Test
+	public void asyncConsumption() {
+		context.publishEvent(new CustomEvent("asyncConsumption"));
+
+		Awaitility.await().atMost(Durations.ONE_SECOND)
+				.untilAsserted(() -> assertThat(assertThat(this.applicationEvents.stream(CustomEvent.class))
+						.singleElement()
+						.extracting(CustomEvent::getMessage, InstanceOfAssertFactories.STRING)
+						.isEqualTo("asyncConsumption")));
+	}
+
+
+	@Configuration
+	static class Config { }
+}