From 4e71981f779cf8c2841f8b5d3ed3124f7006b440 Mon Sep 17 00:00:00 2001 From: "David J. M. Karlsen" Date: Sat, 8 Sep 2018 16:52:07 +0200 Subject: [PATCH] Add Prometheus push gateway support Add support for Prometheus push gateway so that short lived processes (for example batch jobs) can still submit metrics to Prometheus. Closes gh-14353 --- .../pom.xml | 5 + ...metheusMetricsExportAutoConfiguration.java | 120 ++++++++++++++++++ .../prometheus/PrometheusProperties.java | 114 +++++++++++++++++ ...usMetricsExportAutoConfigurationTests.java | 13 ++ .../spring-boot-dependencies/pom.xml | 7 + 5 files changed, 259 insertions(+) diff --git a/spring-boot-project/spring-boot-actuator-autoconfigure/pom.xml b/spring-boot-project/spring-boot-actuator-autoconfigure/pom.xml index 15bbe47126d..4ec774fe71e 100644 --- a/spring-boot-project/spring-boot-actuator-autoconfigure/pom.xml +++ b/spring-boot-project/spring-boot-actuator-autoconfigure/pom.xml @@ -147,6 +147,11 @@ micrometer-registry-prometheus true + + io.prometheus + simpleclient_pushgateway + true + io.micrometer micrometer-registry-signalfx diff --git a/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfiguration.java b/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfiguration.java index d21bf7e46c0..910e3d67b3b 100644 --- a/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfiguration.java +++ b/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfiguration.java @@ -16,10 +16,20 @@ package org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus; +import java.net.UnknownHostException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PreDestroy; + import io.micrometer.core.instrument.Clock; import io.micrometer.prometheus.PrometheusConfig; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.PushGateway; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnEnabledEndpoint; import org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration; @@ -36,6 +46,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; /** * {@link EnableAutoConfiguration Auto-configuration} for exporting metrics to Prometheus. @@ -86,4 +97,113 @@ public class PrometheusMetricsExportAutoConfiguration { } + /** + * Configuration for Prometheus + * Pushgateway. + * + * @author David J. M. Karlsen + */ + @Configuration + @ConditionalOnClass(PushGateway.class) + @ConditionalOnProperty(prefix = "management.metrics.export.prometheus.pushgateway", name = "enabled") + public static class PrometheusPushGatewayConfiguration { + + @Bean + public PushGatewayHandler pushGatewayHandler(CollectorRegistry collectorRegistry, + PrometheusProperties prometheusProperties, Environment environment) { + return new PushGatewayHandler(collectorRegistry, prometheusProperties, + environment); + } + + static class PushGatewayHandler { + + private final Logger logger = LoggerFactory + .getLogger(PrometheusPushGatewayConfiguration.class); + + private final CollectorRegistry collectorRegistry; + + private final PrometheusProperties.PushgatewayProperties pushgatewayProperties; + + private final PushGateway pushGateway; + + private final Environment environment; + + private final ScheduledExecutorService executorService; + + PushGatewayHandler(CollectorRegistry collectorRegistry, + PrometheusProperties prometheusProperties, Environment environment) { + this.collectorRegistry = collectorRegistry; + this.pushgatewayProperties = prometheusProperties.getPushgateway(); + this.pushGateway = new PushGateway( + this.pushgatewayProperties.getBaseUrl()); + this.environment = environment; + this.executorService = Executors.newSingleThreadScheduledExecutor((r) -> { + Thread thread = new Thread(r); + thread.setDaemon(true); + thread.setName("micrometer-pushgateway"); + return thread; + }); + this.executorService.scheduleAtFixedRate(this::push, 0, + this.pushgatewayProperties.getPushRate().toMillis(), + TimeUnit.MILLISECONDS); + } + + void push() { + try { + this.pushGateway.pushAdd(this.collectorRegistry, getJobName(), + this.pushgatewayProperties.getGroupingKeys()); + } + catch (UnknownHostException ex) { + this.logger.error("Unable to locate host '" + + this.pushgatewayProperties.getBaseUrl() + + "'. No longer attempting metrics publication to this host"); + this.executorService.shutdown(); + } + catch (Throwable throwable) { + this.logger.error("Unable to push metrics to Prometheus Pushgateway", + throwable); + } + } + + @PreDestroy + void shutdown() { + this.executorService.shutdown(); + if (this.pushgatewayProperties.isPushOnShutdown()) { + push(); + } + if (this.pushgatewayProperties.isDeleteOnShutdown()) { + delete(); + } + } + + private void delete() { + try { + this.pushGateway.delete(getJobName(), + this.pushgatewayProperties.getGroupingKeys()); + } + catch (Throwable throwable) { + this.logger.error( + "Unable to delete metrics from Prometheus Pushgateway", + throwable); + } + } + + private String getJobName() { + String job = this.pushgatewayProperties.getJob(); + if (job == null) { + job = this.environment.getProperty("spring.application.name"); + } + if (job == null) { + // There's a history of Prometheus spring integration defaulting the + // getJobName name to "spring" from when + // Prometheus integration didn't exist in Spring itself. + job = "spring"; + } + return job; + } + + } + + } + } diff --git a/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusProperties.java b/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusProperties.java index 3612eee7300..a1650e1937d 100644 --- a/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusProperties.java +++ b/spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusProperties.java @@ -17,6 +17,8 @@ package org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus; import java.time.Duration; +import java.util.HashMap; +import java.util.Map; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -36,6 +38,12 @@ public class PrometheusProperties { */ private boolean descriptions = true; + /** + * Configuration options for using Prometheus Pushgateway, allowing metrics to be + * pushed when they cannot be scraped. + */ + private PushgatewayProperties pushgateway = new PushgatewayProperties(); + /** * Step size (i.e. reporting frequency) to use. */ @@ -57,4 +65,110 @@ public class PrometheusProperties { this.step = step; } + public PushgatewayProperties getPushgateway() { + return this.pushgateway; + } + + public void setPushgateway(PushgatewayProperties pushgateway) { + this.pushgateway = pushgateway; + } + + /** + * Configuration options for push-based interaction with Prometheus. + */ + public static class PushgatewayProperties { + + /** + * Enable publishing via a Prometheus Pushgateway. + */ + private Boolean enabled = false; + + /** + * Required host:port or ip:port of the Pushgateway. + */ + private String baseUrl = "localhost:9091"; + + /** + * Required identifier for this application instance. + */ + private String job; + + /** + * Frequency with which to push metrics to Pushgateway. + */ + private Duration pushRate = Duration.ofMinutes(1); + + /** + * Push metrics right before shut-down. Mostly useful for batch jobs. + */ + private boolean pushOnShutdown = true; + + /** + * Delete metrics from Pushgateway when application is shut-down. + */ + private boolean deleteOnShutdown = true; + + /** + * Used to group metrics in pushgateway. A common example is setting + */ + private Map groupingKeys = new HashMap<>(); + + public Boolean getEnabled() { + return this.enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } + + public String getBaseUrl() { + return this.baseUrl; + } + + public void setBaseUrl(String baseUrl) { + this.baseUrl = baseUrl; + } + + public String getJob() { + return this.job; + } + + public void setJob(String job) { + this.job = job; + } + + public Duration getPushRate() { + return this.pushRate; + } + + public void setPushRate(Duration pushRate) { + this.pushRate = pushRate; + } + + public boolean isPushOnShutdown() { + return this.pushOnShutdown; + } + + public void setPushOnShutdown(boolean pushOnShutdown) { + this.pushOnShutdown = pushOnShutdown; + } + + public boolean isDeleteOnShutdown() { + return this.deleteOnShutdown; + } + + public void setDeleteOnShutdown(boolean deleteOnShutdown) { + this.deleteOnShutdown = deleteOnShutdown; + } + + public Map getGroupingKeys() { + return this.groupingKeys; + } + + public void setGroupingKeys(Map groupingKeys) { + this.groupingKeys = groupingKeys; + } + + } + } diff --git a/spring-boot-project/spring-boot-actuator-autoconfigure/src/test/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfigurationTests.java b/spring-boot-project/spring-boot-actuator-autoconfigure/src/test/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfigurationTests.java index 6d89f6c315d..56155d6739b 100644 --- a/spring-boot-project/spring-boot-actuator-autoconfigure/src/test/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfigurationTests.java +++ b/spring-boot-project/spring-boot-actuator-autoconfigure/src/test/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfigurationTests.java @@ -22,6 +22,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.CollectorRegistry; import org.junit.Test; +import org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus.PrometheusMetricsExportAutoConfiguration.PrometheusPushGatewayConfiguration; import org.springframework.boot.actuate.autoconfigure.web.server.ManagementContextAutoConfiguration; import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusScrapeEndpoint; import org.springframework.boot.autoconfigure.AutoConfigurations; @@ -128,6 +129,18 @@ public class PrometheusMetricsExportAutoConfigurationTests { .hasSingleBean(PrometheusScrapeEndpoint.class)); } + @Test + public void withPushGatewayEnabled() { + this.contextRunner + .withConfiguration( + AutoConfigurations.of(ManagementContextAutoConfiguration.class)) + .withPropertyValues( + "management.metrics.export.prometheus.pushgateway.enabled=true") + .withUserConfiguration(BaseConfiguration.class) + .run((context) -> assertThat(context).hasSingleBean( + PrometheusPushGatewayConfiguration.PushGatewayHandler.class)); + } + @Configuration static class BaseConfiguration { diff --git a/spring-boot-project/spring-boot-dependencies/pom.xml b/spring-boot-project/spring-boot-dependencies/pom.xml index da05b0973ee..e85932c32d3 100644 --- a/spring-boot-project/spring-boot-dependencies/pom.xml +++ b/spring-boot-project/spring-boot-dependencies/pom.xml @@ -143,6 +143,8 @@ 1.1.0 1.0.3 42.2.5 + + 0.5.0 2.3.0 4.2.1 5.4.1 @@ -989,6 +991,11 @@ netty-tcnative-boringssl-static ${netty-tcnative.version} + + io.prometheus + simpleclient_pushgateway + ${prometheus-pushgateway.version} + io.projectreactor reactor-bom