From 249620246f07f691c0dc79f2e219246b5f3a3340 Mon Sep 17 00:00:00 2001 From: Tommy Karlsson Date: Mon, 18 Aug 2025 15:40:14 +0200 Subject: [PATCH 1/2] Improve Hazelcast health indicator to check that Hazelcast is running This commit improves the Hazelcast health indicator to fail if the Lifecycle manager indicates that the instance is no longer running. This is needed because when the default Out-Of-Memory handler in Hazelcast detects a OOM error, and terminates Hazelcast, it does so on the node but not via the Hazelcast lifecycle manager - and the results is that it is still possible to successfully execute transactions in Hazelcast after this termination. The health indicator should not report UP after Hazelcast has detected an OOM and terminated itself. See gh-46877 Signed-off-by: Tommy Karlsson --- .../hazelcast/HazelcastHealthIndicator.java | 5 ++ .../HazelcastHealthIndicatorTests.java | 59 ++++++++++++++----- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicator.java b/spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicator.java index ee6fffc0e33..a284cd502e4 100644 --- a/spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicator.java +++ b/spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicator.java @@ -28,6 +28,7 @@ import org.springframework.util.Assert; * * @author Dmytro Nosan * @author Stephane Nicoll + * @author Tommy Karlsson * @since 2.2.0 */ public class HazelcastHealthIndicator extends AbstractHealthIndicator { @@ -42,6 +43,10 @@ public class HazelcastHealthIndicator extends AbstractHealthIndicator { @Override protected void doHealthCheck(Health.Builder builder) { + if (!this.hazelcast.getLifecycleService().isRunning()) { + builder.down(); + return; + } this.hazelcast.executeTransaction((context) -> { String uuid = this.hazelcast.getLocalEndpoint().getUuid().toString(); builder.up().withDetail("name", this.hazelcast.getName()).withDetail("uuid", uuid); diff --git a/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java b/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java index e41e0cd09f4..e0caf376d6c 100644 --- a/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java +++ b/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java @@ -18,6 +18,8 @@ package org.springframework.boot.actuate.hazelcast; import com.hazelcast.core.HazelcastException; import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.instance.impl.HazelcastInstanceProxy; +import com.hazelcast.instance.impl.OutOfMemoryHandlerHelper; import org.junit.jupiter.api.Test; import org.springframework.boot.actuate.health.Health; @@ -37,25 +39,26 @@ import static org.mockito.Mockito.mock; * * @author Dmytro Nosan * @author Stephane Nicoll + * @author Tommy Karlsson */ +@WithResource(name = "hazelcast.xml", content = """ + + actuator-hazelcast + + + + + + + + + """) class HazelcastHealthIndicatorTests { @Test - @WithResource(name = "hazelcast.xml", content = """ - - actuator-hazelcast - - - - - - - - - """) void hazelcastUp() { new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class)) .withPropertyValues("spring.hazelcast.config=hazelcast.xml") @@ -69,6 +72,32 @@ class HazelcastHealthIndicatorTests { }); } + @Test + void hazelcastShutdown() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class)) + .withPropertyValues("spring.hazelcast.config=hazelcast.xml") + .run((context) -> { + HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class); + hazelcast.shutdown(); + Health health = new HazelcastHealthIndicator(hazelcast).health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + }); + } + + @Test + void hazelcastOOMShutdown() { + new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class)) + .withPropertyValues("spring.hazelcast.config=hazelcast.xml") + .run((context) -> { + HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class); + HazelcastInstance original = ((HazelcastInstanceProxy) hazelcast).getOriginal(); + OutOfMemoryHandlerHelper.tryCloseConnections(original); + OutOfMemoryHandlerHelper.tryShutdown(original); + Health health = new HazelcastHealthIndicator(hazelcast).health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + }); + } + @Test void hazelcastDown() { HazelcastInstance hazelcast = mock(HazelcastInstance.class); From 375b0b89f10355c2d9c729d49675c14d7c775c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Nicoll?= Date: Tue, 19 Aug 2025 16:32:42 +0200 Subject: [PATCH 2/2] Polish contribution See gh-46877 --- .../HazelcastHealthIndicatorTests.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java b/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java index e0caf376d6c..74042e2173e 100644 --- a/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java +++ b/spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java @@ -18,8 +18,7 @@ package org.springframework.boot.actuate.hazelcast; import com.hazelcast.core.HazelcastException; import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.instance.impl.HazelcastInstanceProxy; -import com.hazelcast.instance.impl.OutOfMemoryHandlerHelper; +import com.hazelcast.core.LifecycleService; import org.junit.jupiter.api.Test; import org.springframework.boot.actuate.health.Health; @@ -32,6 +31,7 @@ import org.springframework.boot.testsupport.classpath.resources.WithResource; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; import static org.mockito.Mockito.mock; /** @@ -85,25 +85,31 @@ class HazelcastHealthIndicatorTests { } @Test - void hazelcastOOMShutdown() { - new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class)) - .withPropertyValues("spring.hazelcast.config=hazelcast.xml") - .run((context) -> { - HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class); - HazelcastInstance original = ((HazelcastInstanceProxy) hazelcast).getOriginal(); - OutOfMemoryHandlerHelper.tryCloseConnections(original); - OutOfMemoryHandlerHelper.tryShutdown(original); - Health health = new HazelcastHealthIndicator(hazelcast).health(); - assertThat(health.getStatus()).isEqualTo(Status.DOWN); - }); + void hazelcastLifecycleNotRunning() { + HazelcastInstance hazelcast = mockHazelcastInstance(false); + Health health = new HazelcastHealthIndicator(hazelcast).health(); + assertThat(health.getStatus()).isEqualTo(Status.DOWN); + then(hazelcast).should().getLifecycleService(); + then(hazelcast).shouldHaveNoMoreInteractions(); } @Test void hazelcastDown() { - HazelcastInstance hazelcast = mock(HazelcastInstance.class); + HazelcastInstance hazelcast = mockHazelcastInstance(true); given(hazelcast.executeTransaction(any())).willThrow(new HazelcastException()); Health health = new HazelcastHealthIndicator(hazelcast).health(); assertThat(health.getStatus()).isEqualTo(Status.DOWN); + then(hazelcast).should().getLifecycleService(); + then(hazelcast).should().executeTransaction(any()); + then(hazelcast).shouldHaveNoMoreInteractions(); + } + + private static HazelcastInstance mockHazelcastInstance(boolean isRunning) { + LifecycleService lifecycleService = mock(LifecycleService.class); + given(lifecycleService.isRunning()).willReturn(isRunning); + HazelcastInstance hazelcastInstance = mock(HazelcastInstance.class); + given(hazelcastInstance.getLifecycleService()).willReturn(lifecycleService); + return hazelcastInstance; } }