Add Prometheus push gateway support

Add support for Prometheus push gateway so that short lived processes
(for example batch jobs) can still submit metrics to Prometheus.

Closes gh-14353

spring-boot-project/spring-boot-actuator-autoconfigure/pom.xml

@@ -147,6 +147,11 @@
 			<artifactId>micrometer-registry-prometheus</artifactId>
 			<optional>true</optional>
 		</dependency>
+		<dependency>
+			<groupId>io.prometheus</groupId>
+			<artifactId>simpleclient_pushgateway</artifactId>
+			<optional>true</optional>
+		</dependency>
 		<dependency>
 			<groupId>io.micrometer</groupId>
 			<artifactId>micrometer-registry-signalfx</artifactId>

spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfiguration.java

@@ -16,10 +16,20 @@
 
 package org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus;
 
+import java.net.UnknownHostException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.PreDestroy;
+
 import io.micrometer.core.instrument.Clock;
 import io.micrometer.prometheus.PrometheusConfig;
 import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.exporter.PushGateway;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnEnabledEndpoint;
 import org.springframework.boot.actuate.autoconfigure.metrics.CompositeMeterRegistryAutoConfiguration;
@@ -36,6 +46,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
 
 /**
  * {@link EnableAutoConfiguration Auto-configuration} for exporting metrics to Prometheus.
@@ -86,4 +97,113 @@ public class PrometheusMetricsExportAutoConfiguration {
 
 	}
 
+	/**
+	 * Configuration for <a href="https://github.com/prometheus/pushgateway">Prometheus
+	 * Pushgateway</a>.
+	 *
+	 * @author David J. M. Karlsen
+	 */
+	@Configuration
+	@ConditionalOnClass(PushGateway.class)
+	@ConditionalOnProperty(prefix = "management.metrics.export.prometheus.pushgateway", name = "enabled")
+	public static class PrometheusPushGatewayConfiguration {
+
+		@Bean
+		public PushGatewayHandler pushGatewayHandler(CollectorRegistry collectorRegistry,
+				PrometheusProperties prometheusProperties, Environment environment) {
+			return new PushGatewayHandler(collectorRegistry, prometheusProperties,
+					environment);
+		}
+
+		static class PushGatewayHandler {
+
+			private final Logger logger = LoggerFactory
+					.getLogger(PrometheusPushGatewayConfiguration.class);
+
+			private final CollectorRegistry collectorRegistry;
+
+			private final PrometheusProperties.PushgatewayProperties pushgatewayProperties;
+
+			private final PushGateway pushGateway;
+
+			private final Environment environment;
+
+			private final ScheduledExecutorService executorService;
+
+			PushGatewayHandler(CollectorRegistry collectorRegistry,
+					PrometheusProperties prometheusProperties, Environment environment) {
+				this.collectorRegistry = collectorRegistry;
+				this.pushgatewayProperties = prometheusProperties.getPushgateway();
+				this.pushGateway = new PushGateway(
+						this.pushgatewayProperties.getBaseUrl());
+				this.environment = environment;
+				this.executorService = Executors.newSingleThreadScheduledExecutor((r) -> {
+					Thread thread = new Thread(r);
+					thread.setDaemon(true);
+					thread.setName("micrometer-pushgateway");
+					return thread;
+				});
+				this.executorService.scheduleAtFixedRate(this::push, 0,
+						this.pushgatewayProperties.getPushRate().toMillis(),
+						TimeUnit.MILLISECONDS);
+			}
+
+			void push() {
+				try {
+					this.pushGateway.pushAdd(this.collectorRegistry, getJobName(),
+							this.pushgatewayProperties.getGroupingKeys());
+				}
+				catch (UnknownHostException ex) {
+					this.logger.error("Unable to locate host '"
+							+ this.pushgatewayProperties.getBaseUrl()
+							+ "'. No longer attempting metrics publication to this host");
+					this.executorService.shutdown();
+				}
+				catch (Throwable throwable) {
+					this.logger.error("Unable to push metrics to Prometheus Pushgateway",
+							throwable);
+				}
+			}
+
+			@PreDestroy
+			void shutdown() {
+				this.executorService.shutdown();
+				if (this.pushgatewayProperties.isPushOnShutdown()) {
+					push();
+				}
+				if (this.pushgatewayProperties.isDeleteOnShutdown()) {
+					delete();
+				}
+			}
+
+			private void delete() {
+				try {
+					this.pushGateway.delete(getJobName(),
+							this.pushgatewayProperties.getGroupingKeys());
+				}
+				catch (Throwable throwable) {
+					this.logger.error(
+							"Unable to delete metrics from Prometheus Pushgateway",
+							throwable);
+				}
+			}
+
+			private String getJobName() {
+				String job = this.pushgatewayProperties.getJob();
+				if (job == null) {
+					job = this.environment.getProperty("spring.application.name");
+				}
+				if (job == null) {
+					// There's a history of Prometheus spring integration defaulting the
+					// getJobName name to "spring" from when
+					// Prometheus integration didn't exist in Spring itself.
+					job = "spring";
+				}
+				return job;
+			}
+
+		}
+
+	}
+
 }

spring-boot-project/spring-boot-actuator-autoconfigure/src/main/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusProperties.java

@@ -17,6 +17,8 @@
 package org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus;
 
 import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
@@ -36,6 +38,12 @@ public class PrometheusProperties {
 	 */
 	private boolean descriptions = true;
 
+	/**
+	 * Configuration options for using Prometheus Pushgateway, allowing metrics to be
+	 * pushed when they cannot be scraped.
+	 */
+	private PushgatewayProperties pushgateway = new PushgatewayProperties();
+
 	/**
 	 * Step size (i.e. reporting frequency) to use.
 	 */
@@ -57,4 +65,110 @@ public class PrometheusProperties {
 		this.step = step;
 	}
 
+	public PushgatewayProperties getPushgateway() {
+		return this.pushgateway;
+	}
+
+	public void setPushgateway(PushgatewayProperties pushgateway) {
+		this.pushgateway = pushgateway;
+	}
+
+	/**
+	 * Configuration options for push-based interaction with Prometheus.
+	 */
+	public static class PushgatewayProperties {
+
+		/**
+		 * Enable publishing via a Prometheus Pushgateway.
+		 */
+		private Boolean enabled = false;
+
+		/**
+		 * Required host:port or ip:port of the Pushgateway.
+		 */
+		private String baseUrl = "localhost:9091";
+
+		/**
+		 * Required identifier for this application instance.
+		 */
+		private String job;
+
+		/**
+		 * Frequency with which to push metrics to Pushgateway.
+		 */
+		private Duration pushRate = Duration.ofMinutes(1);
+
+		/**
+		 * Push metrics right before shut-down. Mostly useful for batch jobs.
+		 */
+		private boolean pushOnShutdown = true;
+
+		/**
+		 * Delete metrics from Pushgateway when application is shut-down.
+		 */
+		private boolean deleteOnShutdown = true;
+
+		/**
+		 * Used to group metrics in pushgateway. A common example is setting
+		 */
+		private Map<String, String> groupingKeys = new HashMap<>();
+
+		public Boolean getEnabled() {
+			return this.enabled;
+		}
+
+		public void setEnabled(Boolean enabled) {
+			this.enabled = enabled;
+		}
+
+		public String getBaseUrl() {
+			return this.baseUrl;
+		}
+
+		public void setBaseUrl(String baseUrl) {
+			this.baseUrl = baseUrl;
+		}
+
+		public String getJob() {
+			return this.job;
+		}
+
+		public void setJob(String job) {
+			this.job = job;
+		}
+
+		public Duration getPushRate() {
+			return this.pushRate;
+		}
+
+		public void setPushRate(Duration pushRate) {
+			this.pushRate = pushRate;
+		}
+
+		public boolean isPushOnShutdown() {
+			return this.pushOnShutdown;
+		}
+
+		public void setPushOnShutdown(boolean pushOnShutdown) {
+			this.pushOnShutdown = pushOnShutdown;
+		}
+
+		public boolean isDeleteOnShutdown() {
+			return this.deleteOnShutdown;
+		}
+
+		public void setDeleteOnShutdown(boolean deleteOnShutdown) {
+			this.deleteOnShutdown = deleteOnShutdown;
+		}
+
+		public Map<String, String> getGroupingKeys() {
+			return this.groupingKeys;
+		}
+
+		public void setGroupingKeys(Map<String, String> groupingKeys) {
+			this.groupingKeys = groupingKeys;
+		}
+
+	}
+
 }

spring-boot-project/spring-boot-actuator-autoconfigure/src/test/java/org/springframework/boot/actuate/autoconfigure/metrics/export/prometheus/PrometheusMetricsExportAutoConfigurationTests.java

@@ -22,6 +22,7 @@ import io.micrometer.prometheus.PrometheusMeterRegistry;
 import io.prometheus.client.CollectorRegistry;
 import org.junit.Test;
 
+import org.springframework.boot.actuate.autoconfigure.metrics.export.prometheus.PrometheusMetricsExportAutoConfiguration.PrometheusPushGatewayConfiguration;
 import org.springframework.boot.actuate.autoconfigure.web.server.ManagementContextAutoConfiguration;
 import org.springframework.boot.actuate.metrics.export.prometheus.PrometheusScrapeEndpoint;
 import org.springframework.boot.autoconfigure.AutoConfigurations;
@@ -128,6 +129,18 @@ public class PrometheusMetricsExportAutoConfigurationTests {
 						.hasSingleBean(PrometheusScrapeEndpoint.class));
 	}
 
+	@Test
+	public void withPushGatewayEnabled() {
+		this.contextRunner
+				.withConfiguration(
+						AutoConfigurations.of(ManagementContextAutoConfiguration.class))
+				.withPropertyValues(
+						"management.metrics.export.prometheus.pushgateway.enabled=true")
+				.withUserConfiguration(BaseConfiguration.class)
+				.run((context) -> assertThat(context).hasSingleBean(
+						PrometheusPushGatewayConfiguration.PushGatewayHandler.class));
+	}
+
 	@Configuration
 	static class BaseConfiguration {
 

spring-boot-project/spring-boot-dependencies/pom.xml

@@ -143,6 +143,8 @@
 		<nio-multipart-parser.version>1.1.0</nio-multipart-parser.version>
 		<pooled-jms-version>1.0.3</pooled-jms-version>
 		<postgresql.version>42.2.5</postgresql.version>
+		<!-- need to take care that this version harmonizes with micrometer ones -->
+		<prometheus-pushgateway.version>0.5.0</prometheus-pushgateway.version>
 		<quartz.version>2.3.0</quartz.version>
 		<querydsl.version>4.2.1</querydsl.version>
 		<rabbit-amqp-client.version>5.4.1</rabbit-amqp-client.version>
@@ -989,6 +991,11 @@
 				<artifactId>netty-tcnative-boringssl-static</artifactId>
 				<version>${netty-tcnative.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>io.prometheus</groupId>
+				<artifactId>simpleclient_pushgateway</artifactId>
+				<version>${prometheus-pushgateway.version}</version>
+			</dependency>
 			<dependency>
 				<groupId>io.projectreactor</groupId>
 				<artifactId>reactor-bom</artifactId>