Make AbstractTestNGSpringContextTests thread-safe regarding tracked exceptions

Prior to this commit, AbstractTestNGSpringContextTests was not
thread-safe with regard to tracked exceptions.

To address that, AbstractTestNGSpringContextTests now tracks the test
exception via a ThreadLocal.

Closes gh-35528

spring-test/src/main/java/org/springframework/test/context/testng/AbstractTestNGSpringContextTests.java

@@ -74,8 +74,7 @@ public abstract class AbstractTestNGSpringContextTests implements IHookable, App
 
 	private final TestContextManager testContextManager;
 
-	@Nullable
-	private Throwable testException;
+	private final ThreadLocal<Throwable> testException = new ThreadLocal<>();
 
 
 	/**
@@ -141,31 +140,33 @@ public abstract class AbstractTestNGSpringContextTests implements IHookable, App
 	public void run(IHookCallBack callBack, ITestResult testResult) {
 		Method testMethod = testResult.getMethod().getConstructorOrMethod().getMethod();
 		boolean beforeCallbacksExecuted = false;
+		Throwable currentException = null;
 
 		try {
 			this.testContextManager.beforeTestExecution(this, testMethod);
 			beforeCallbacksExecuted = true;
 		}
 		catch (Throwable ex) {
-			this.testException = ex;
+			currentException = ex;
 		}
 
 		if (beforeCallbacksExecuted) {
 			callBack.runTestMethod(testResult);
-			this.testException = getTestResultException(testResult);
+			currentException = getTestResultException(testResult);
 		}
 
 		try {
-			this.testContextManager.afterTestExecution(this, testMethod, this.testException);
+			this.testContextManager.afterTestExecution(this, testMethod, currentException);
 		}
 		catch (Throwable ex) {
-			if (this.testException == null) {
-				this.testException = ex;
+			if (currentException == null) {
+				currentException = ex;
 			}
 		}
 
-		if (this.testException != null) {
-			throwAsUncheckedException(this.testException);
+		if (currentException != null) {
+			this.testException.set(currentException);
+			throwAsUncheckedException(currentException);
 		}
 	}
 
@@ -180,10 +181,10 @@ public abstract class AbstractTestNGSpringContextTests implements IHookable, App
 	@AfterMethod(alwaysRun = true)
 	protected void springTestContextAfterTestMethod(Method testMethod) throws Exception {
 		try {
-			this.testContextManager.afterTestMethod(this, testMethod, this.testException);
+			this.testContextManager.afterTestMethod(this, testMethod, this.testException.get());
 		}
 		finally {
-			this.testException = null;
+			this.testException.remove();
 		}
 	}
 

spring-test/src/test/java/org/springframework/test/context/cache/ClassLevelDirtiesContextTestNGTests.java

@@ -145,9 +145,9 @@ class ClassLevelDirtiesContextTestNGTests {
 		testNG.setVerbose(0);
 		testNG.run();
 
-		assertThat(listener.testFailureCount).as("Failures for test class [" + testClass + "].").isEqualTo(expectedTestFailureCount);
-		assertThat(listener.testStartCount).as("Tests started for test class [" + testClass + "].").isEqualTo(expectedTestStartedCount);
-		assertThat(listener.testSuccessCount).as("Successful tests for test class [" + testClass + "].").isEqualTo(expectedTestFinishedCount);
+		assertThat(listener.testFailureCount.get()).as("Failures for test class [" + testClass + "].").isEqualTo(expectedTestFailureCount);
+		assertThat(listener.testStartCount.get()).as("Tests started for test class [" + testClass + "].").isEqualTo(expectedTestStartedCount);
+		assertThat(listener.testSuccessCount.get()).as("Successful tests for test class [" + testClass + "].").isEqualTo(expectedTestFinishedCount);
 	}
 
 	private void assertBehaviorForCleanTestCase() {

spring-test/src/test/java/org/springframework/test/context/testng/FailingBeforeAndAfterMethodsTestNGTests.java

@@ -64,10 +64,10 @@ class FailingBeforeAndAfterMethodsTestNGTests {
 
 		String name = clazz.getSimpleName();
 
-		assertThat(listener.testStartCount).as("tests started for [" + name + "] ==> ").isEqualTo(expectedTestStartCount);
-		assertThat(listener.testSuccessCount).as("successful tests for [" + name + "] ==> ").isEqualTo(expectedTestSuccessCount);
-		assertThat(listener.testFailureCount).as("failed tests for [" + name + "] ==> ").isEqualTo(expectedFailureCount);
-		assertThat(listener.failedConfigurationsCount).as("failed configurations for [" + name + "] ==> ").isEqualTo(expectedFailedConfigurationsCount);
+		assertThat(listener.testStartCount.get()).as("tests started for [" + name + "] ==> ").isEqualTo(expectedTestStartCount);
+		assertThat(listener.testSuccessCount.get()).as("successful tests for [" + name + "] ==> ").isEqualTo(expectedTestSuccessCount);
+		assertThat(listener.testFailureCount.get()).as("failed tests for [" + name + "] ==> ").isEqualTo(expectedFailureCount);
+		assertThat(listener.failedConfigurationsCount.get()).as("failed configurations for [" + name + "] ==> ").isEqualTo(expectedFailedConfigurationsCount);
 	}
 
 	static List<Arguments> testData() {

spring-test/src/test/java/org/springframework/test/context/testng/TestNGConcurrencyTests.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2002-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.test.context.testng;
+
+import org.testng.TestNG;
+import org.testng.annotations.Test;
+import org.testng.xml.XmlSuite.ParallelMode;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.ContextConfiguration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration tests for concurrent TestNG tests.
+ *
+ * @author Sam Brannen
+ * @since 6.2.12
+ * @see <a href="https://github.com/spring-projects/spring-framework/issues/35528">gh-35528</a>
+ */
+class TestNGConcurrencyTests {
+
+	@org.junit.jupiter.api.Test
+	void runTestsInParallel() throws Exception {
+		TrackingTestNGTestListener listener = new TrackingTestNGTestListener();
+
+		TestNG testNG = new TestNG();
+		testNG.addListener(listener);
+		testNG.setTestClasses(new Class<?>[] { ConcurrentTestCase.class });
+		testNG.setParallel(ParallelMode.METHODS);
+		testNG.setThreadCount(5);
+		testNG.setVerbose(0);
+		testNG.run();
+
+		assertThat(listener.testStartCount.get()).as("tests started").isEqualTo(10);
+		assertThat(listener.testSuccessCount.get()).as("successful tests").isEqualTo(10);
+		assertThat(listener.testFailureCount.get()).as("failed tests").isEqualTo(0);
+		assertThat(listener.failedConfigurationsCount.get()).as("failed configurations").isEqualTo(0);
+		assertThat(listener.throwables).isEmpty();
+	}
+
+
+	@ContextConfiguration
+	static class ConcurrentTestCase extends AbstractTestNGSpringContextTests {
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message1")
+		public void message1() {
+			throw new RuntimeException("Message1");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message2")
+		public void message2() {
+			throw new RuntimeException("Message2");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message3")
+		public void message3() {
+			throw new RuntimeException("Message3");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message4")
+		public void message4() {
+			throw new RuntimeException("Message4");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message5")
+		public void message5() {
+			throw new RuntimeException("Message5");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message6")
+		public void message6() {
+			throw new RuntimeException("Message6");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message7")
+		public void message7() {
+			throw new RuntimeException("Message7");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message8")
+		public void message8() {
+			throw new RuntimeException("Message8");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message9")
+		public void message9() {
+			throw new RuntimeException("Message9");
+		}
+
+		@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Message10")
+		public void message10() {
+			throw new RuntimeException("Message10");
+		}
+
+
+		@Configuration
+		static class Config {
+		}
+
+	}
+
+}

spring-test/src/test/java/org/springframework/test/context/testng/TestNGTestSuite.java

@@ -16,6 +16,7 @@
 
 package org.springframework.test.context.testng;
 
+import org.junit.platform.suite.api.IncludeClassNamePatterns;
 import org.junit.platform.suite.api.IncludeEngines;
 import org.junit.platform.suite.api.SelectPackages;
 import org.junit.platform.suite.api.Suite;
@@ -40,7 +41,8 @@ import org.junit.platform.suite.api.Suite;
  * @since 5.3.11
  */
 @Suite
-@IncludeEngines("testng")
+@IncludeEngines({"testng", "junit-jupiter"})
 @SelectPackages("org.springframework.test.context.testng")
+@IncludeClassNamePatterns(".*Tests?$")
 class TestNGTestSuite {
 }

spring-test/src/test/java/org/springframework/test/context/testng/TrackingTestNGTestListener.java

@@ -16,6 +16,10 @@
 
 package org.springframework.test.context.testng;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import org.testng.ITestContext;
 import org.testng.ITestListener;
 import org.testng.ITestResult;
@@ -29,18 +33,20 @@ import org.testng.ITestResult;
  */
 public class TrackingTestNGTestListener implements ITestListener {
 
-	public int testStartCount = 0;
+	public final List<Throwable> throwables = new ArrayList<>();
+
+	public final AtomicInteger testStartCount = new AtomicInteger();
 
-	public int testSuccessCount = 0;
+	public final AtomicInteger testSuccessCount = new AtomicInteger();
 
-	public int testFailureCount = 0;
+	public final AtomicInteger testFailureCount = new AtomicInteger();
 
-	public int failedConfigurationsCount = 0;
+	public final AtomicInteger failedConfigurationsCount = new AtomicInteger();
 
 
 	@Override
 	public void onFinish(ITestContext testContext) {
-		this.failedConfigurationsCount += testContext.getFailedConfigurations().size();
+		this.failedConfigurationsCount.addAndGet(testContext.getFailedConfigurations().size());
 	}
 
 	@Override
@@ -53,7 +59,12 @@ public class TrackingTestNGTestListener implements ITestListener {
 
 	@Override
 	public void onTestFailure(ITestResult testResult) {
-		this.testFailureCount++;
+		this.testFailureCount.incrementAndGet();
+
+		Throwable throwable = testResult.getThrowable();
+		if (throwable != null) {
+			this.throwables.add(throwable);
+		}
 	}
 
 	@Override
@@ -62,12 +73,12 @@ public class TrackingTestNGTestListener implements ITestListener {
 
 	@Override
 	public void onTestStart(ITestResult testResult) {
-		this.testStartCount++;
+		this.testStartCount.incrementAndGet();
 	}
 
 	@Override
 	public void onTestSuccess(ITestResult testResult) {
-		this.testSuccessCount++;
+		this.testSuccessCount.incrementAndGet();
 	}
 
 }

src/checkstyle/checkstyle-suppressions.xml

@@ -115,7 +115,7 @@
 	<suppress files="src[\\/]test[\\/]java[\\/]org[\\/]springframework[\\/]test[\\/]web[\\/](client|reactive|servlet)[\\/].+Tests" checks="IllegalImport" id="bannedHamcrestImports"/>
 	<suppress files="src[\\/]test[\\/]java[\\/]org[\\/]springframework[\\/]test[\\/]context[\\/](aot|junit4)" checks="SpringJUnit5"/>
 	<suppress files="AutowiredConfigurationErrorsIntegrationTests" checks="SpringJUnit5" message="Lifecycle method .+ should not be private"/>
-	<suppress files="org[\\/]springframework[\\/]test[\\/]context[\\/].+[\\/](ExpectedExceptionSpringRunnerTests|StandardJUnit4FeaturesTests|ProgrammaticTxMgmtTestNGTests)" checks="RegexpSinglelineJava" id="expectedExceptionAnnotation"/>
+	<suppress files="org[\\/]springframework[\\/]test[\\/]context[\\/].+[\\/](ExpectedExceptionSpringRunnerTests|StandardJUnit4FeaturesTests|TestNGConcurrencyTests|ProgrammaticTxMgmtTestNGTests)" checks="RegexpSinglelineJava" id="expectedExceptionAnnotation"/>
 
 	<!-- spring-web -->
 	<suppress files="SpringHandlerInstantiator" checks="JavadocStyle"/>