From c0ccdc24a0b96f56bd4985863a73cc6bb4ed9950 Mon Sep 17 00:00:00 2001 From: Brian Clozel Date: Tue, 28 Apr 2020 14:25:04 +0200 Subject: [PATCH] Add Concourse Build pipeline This commit adds the Concourse build infrastructure under the "ci/" folder. It contains: * "images/", the definitions of container images used in the pipeline * "pipeline.yml" the build pipeline definition * "parameters.yml" the parameters for the pipeline definition * "tasks/" the tasks used by the build pipeline * "scripts/" scripts used by pipeline tasks This first iteration of the Concourse pipeline builds CI container images for JDK8, JDK11, JDK14 and JDK15. These images are used for building and publishing the master branch of Spring Framework using JDK8. Other JDK variants are sanity checks and don't actually publish the produced artifacts. See gh-22490 --- README.md | 2 +- ci/README.adoc | 51 +++ ci/images/README.adoc | 21 ++ ci/images/get-jdk-url.sh | 20 ++ ci/images/setup.sh | 31 ++ .../spring-framework-ci-image/Dockerfile | 8 + .../Dockerfile | 8 + .../Dockerfile | 8 + .../Dockerfile | 8 + ci/parameters.yml | 14 + ci/pipeline.yml | 323 ++++++++++++++++++ ci/scripts/build-project.sh | 9 + ci/scripts/common.sh | 2 + ci/tasks/build-project.yml | 21 ++ 14 files changed, 525 insertions(+), 1 deletion(-) create mode 100644 ci/README.adoc create mode 100644 ci/images/README.adoc create mode 100755 ci/images/get-jdk-url.sh create mode 100755 ci/images/setup.sh create mode 100644 ci/images/spring-framework-ci-image/Dockerfile create mode 100644 ci/images/spring-framework-jdk11-ci-image/Dockerfile create mode 100644 ci/images/spring-framework-jdk14-ci-image/Dockerfile create mode 100644 ci/images/spring-framework-jdk15-ci-image/Dockerfile create mode 100644 ci/parameters.yml create mode 100644 ci/pipeline.yml create mode 100755 ci/scripts/build-project.sh create mode 100644 ci/scripts/common.sh create mode 100644 ci/tasks/build-project.yml diff --git a/README.md b/README.md index 32aa262d732..93b3c486667 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Spring Framework [![Build Status](https://build.spring.io/plugins/servlet/wittified/build-status/SPR-PUBM)](https://build.spring.io/browse/SPR) +# Spring Framework [![Build Status](https://ci.spring.io/api/v1/teams/spring-framework/pipelines/spring-framework-5.3.x/jobs/build/badge)](https://ci.spring.io/teams/spring-framework/pipelines/spring-framework-5.3.x?groups=Build") This is the home of the Spring Framework: the foundation for all [Spring projects](https://spring.io/projects). Collectively the Spring Framework and the family of Spring projects are often referred to simply as "Spring". diff --git a/ci/README.adoc b/ci/README.adoc new file mode 100644 index 00000000000..c2c154acca9 --- /dev/null +++ b/ci/README.adoc @@ -0,0 +1,51 @@ +== Spring Framework Concourse pipeline + +The Spring Framework is using https://concourse-ci.org/[Concourse] for its CI build and other automated tasks. +The Spring team has a dedicated Concourse instance available at https://ci.spring.io. + +=== Setting up your development environment + +If you're part of the Spring Framework project on GitHub, you can get access to CI management features. +First, you need to go to https://ci.spring.io and install the client CLI for your platform (see bottom right of the screen). + +You can then login with the instance using: + +[source] +---- +$ fly -t spring login -n spring-framework -c https://ci.spring.io +---- + +Once logged in, you should get something like: + +[source] +---- +$ fly ts +name url team expiry +spring https://ci.spring.io spring-framework Wed, 25 Mar 2020 17:45:26 UTC +---- + +=== Pipeline configuration and structure +The build pipelines are described in `pipeline.yml` file. +This file is listing Concourse resources, i.e. build inputs and outputs such as container images, artifact repositories, source repositories, notification services, etc. +It also describes jobs (a job is a sequence of inputs, tasks and outputs); jobs are organized by groups. + +The `pipeline.yml` definition contains `((parameters))` which are loaded from the `parameters.yml` file or from our https://docs.cloudfoundry.org/credhub/[credhub instance]. + +You'll find in this folder the following resources: +* `pipeline.yml` the build pipeline +* `parameters.yml` the build parameters used for the pipeline +* `images/` holds the container images definitions used in this pipeline +* `scripts/` holds the build scripts that ship within the CI container images +* `tasks` contains the task definitions used in the main `pipeline.yml` + +=== Updating the build pipeline + +Updating files on the repository is not enough to update the build pipeline, as changes need to be applied. +The pipeline can be deployed using the following command: + +[source] +---- +$ fly -t spring set-pipeline -p spring-framework-5.3.x -c ci/pipeline.yml -l ci/parameters.yml +---- + +NOTE: This assumes that you have credhub integration configured with the appropriate secrets. \ No newline at end of file diff --git a/ci/images/README.adoc b/ci/images/README.adoc new file mode 100644 index 00000000000..6da9addd9ca --- /dev/null +++ b/ci/images/README.adoc @@ -0,0 +1,21 @@ +== CI Images + +These images are used by CI to run the actual builds. + +To build the image locally run the following from this directory: + +---- +$ docker build --no-cache -f /Dockerfile . +---- + +For example + +---- +$ docker build --no-cache -f spring-framework-ci-image/Dockerfile . +---- + +To test run: + +---- +$ docker run -it --entrypoint /bin/bash +---- diff --git a/ci/images/get-jdk-url.sh b/ci/images/get-jdk-url.sh new file mode 100755 index 00000000000..d26fd413cc7 --- /dev/null +++ b/ci/images/get-jdk-url.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +case "$1" in + java8) + echo "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u252b09.tar.gz" + ;; + java11) + echo "https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.7_10.tar.gz" + ;; + java14) + echo "https://github.com/AdoptOpenJDK/openjdk14-binaries/releases/download/jdk-14.0.1%2B7/OpenJDK14U-jdk_x64_linux_hotspot_14.0.1_7.tar.gz" + ;; + java15) + echo "https://download.java.net/java/early_access/jdk15/20/GPL/openjdk-15-ea+20_linux-x64_bin.tar.gz" + ;; + *) + echo $"Unknown java version" + exit 1 +esac diff --git a/ci/images/setup.sh b/ci/images/setup.sh new file mode 100755 index 00000000000..b9eab7adba5 --- /dev/null +++ b/ci/images/setup.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -ex + +########################################################### +# UTILS +########################################################### + +apt-get update +apt-get install --no-install-recommends -y ca-certificates net-tools libxml2-utils git curl libudev1 libxml2-utils iptables iproute2 jq fontconfig +rm -rf /var/lib/apt/lists/* + +curl https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.2/concourse-java.sh > /opt/concourse-java.sh + + +########################################################### +# JAVA +########################################################### +JDK_URL=$( ./get-jdk-url.sh $1 ) + +mkdir -p /opt/openjdk +cd /opt/openjdk +curl -L ${JDK_URL} | tar zx --strip-components=1 +test -f /opt/openjdk/bin/java +test -f /opt/openjdk/bin/javac + +########################################################### +# GRADLE ENTERPRISE +########################################################### +cd / +mkdir ~/.gradle +echo 'systemProp.user.name=concourse' > ~/.gradle/gradle.properties \ No newline at end of file diff --git a/ci/images/spring-framework-ci-image/Dockerfile b/ci/images/spring-framework-ci-image/Dockerfile new file mode 100644 index 00000000000..cfd64e44fea --- /dev/null +++ b/ci/images/spring-framework-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java8 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/spring-framework-jdk11-ci-image/Dockerfile b/ci/images/spring-framework-jdk11-ci-image/Dockerfile new file mode 100644 index 00000000000..df007ebf5d9 --- /dev/null +++ b/ci/images/spring-framework-jdk11-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java11 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/spring-framework-jdk14-ci-image/Dockerfile b/ci/images/spring-framework-jdk14-ci-image/Dockerfile new file mode 100644 index 00000000000..a1dab0b61eb --- /dev/null +++ b/ci/images/spring-framework-jdk14-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java14 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/spring-framework-jdk15-ci-image/Dockerfile b/ci/images/spring-framework-jdk15-ci-image/Dockerfile new file mode 100644 index 00000000000..ada77812c83 --- /dev/null +++ b/ci/images/spring-framework-jdk15-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java15 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/parameters.yml b/ci/parameters.yml new file mode 100644 index 00000000000..3e09f785ebf --- /dev/null +++ b/ci/parameters.yml @@ -0,0 +1,14 @@ +email-server: "smtp.svc.pivotal.io" +email-from: "ci@spring.io" +email-to: ["spring-framework-dev@pivotal.io"] +github-repo: "https://github.com/spring-projects/spring-framework.git" +github-repo-name: "spring-projects/spring-framework" +docker-hub-organization: "springci" +artifactory-server: "https://repo.spring.io" +branch: "master" +build-name: "spring-framework" +pipeline-name: "spring-framework" +concourse-url: "https://ci.spring.io" +bintray-subject: "spring" +bintray-repo: "jars" +task-timeout: 1h00m \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 00000000000..539a355077b --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,323 @@ +resource_types: +- name: artifactory-resource + type: docker-image + source: + repository: springio/artifactory-resource + tag: 0.0.12-SNAPSHOT +- name: github-status-resource + type: docker-image + source: + repository: dpb587/github-status-resource + tag: master +- name: email + type: docker-image + source: + repository: pcfseceng/email-resource + tag: 1.0.19 +- name: slack-notification + type: docker-image + source: + repository: cfcommunity/slack-notification-resource + tag: latest +resources: +- name: git-repo + type: git + icon: github-circle + source: + uri: ((github-repo)) + username: ((github-username)) + password: ((github-password)) + branch: ((branch)) +- name: ci-images-git-repo + type: git + icon: github-circle + source: + uri: ((github-repo)) + branch: ((branch)) + paths: ["ci/images/*"] +- name: spring-framework-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: spring-framework-jdk11-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-jdk11-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: spring-framework-jdk14-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-jdk14-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: spring-framework-jdk15-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-jdk15-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: artifactory-repo + type: artifactory-resource + icon: package-variant + source: + uri: ((artifactory-server)) + username: ((artifactory-username)) + password: ((artifactory-password)) + build_name: ((build-name)) +- name: repo-status-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: build +- name: repo-status-jdk11-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: jdk11-build +- name: repo-status-jdk14-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: jdk14-build +- name: repo-status-jdk15-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: jdk15-build +- name: send-an-email + type: email + source: + smtp: + host: ((email-server)) + port: "25" + anonymous: true + from: ((email-from)) + to: ((email-to)) +- name: slack-alert + type: slack-notification + icon: slack + source: + url: ((slack-webhook-url)) +jobs: +- name: build-spring-framework-ci-images + plan: + - get: ci-images-git-repo + trigger: true + - in_parallel: + - put: spring-framework-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-ci-image/Dockerfile + - put: spring-framework-jdk11-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-jdk11-ci-image/Dockerfile + - put: spring-framework-jdk14-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-jdk14-ci-image/Dockerfile + - put: spring-framework-jdk15-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-jdk15-ci-image/Dockerfile +- name: build + serial: true + public: true + plan: + - get: spring-framework-ci-image + - get: git-repo + trigger: true + - put: repo-status-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-build + params: { state: "success", commit: "git-repo" } + - put: artifactory-repo + params: &artifactory-params + repo: libs-snapshot-local + folder: distribution-repository + build_uri: "https://ci.spring.io/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}" + build_number: "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" + disable_checksum_uploads: true + threads: 8 + artifact_set: + - include: + - "/**/spring-framework-*.zip" + properties: + "zip.name": "spring-framework" + "zip.displayname": "Spring Framework" + "zip.deployed": "false" + - include: + - "/**/spring-framework-*-docs.zip" + properties: + "zip.type": "docs" + - include: + - "/**/spring-framework-*-dist.zip" + properties: + "zip.type": "dist" + - include: + - "/**/spring-framework-*-schema.zip" + properties: + "zip.type": "schema" + get_params: + threads: 8 +- name: jdk11-build + serial: true + public: true + plan: + - get: spring-framework-jdk11-ci-image + - get: git-repo + trigger: true + - put: repo-status-jdk11-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-jdk11-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-jdk11-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] JDK 11 Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-jdk11-build + params: { state: "success", commit: "git-repo" } +- name: jdk14-build + serial: true + public: true + plan: + - get: spring-framework-jdk14-ci-image + - get: git-repo + trigger: true + - put: repo-status-jdk14-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-jdk14-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-jdk14-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] JDK 14 Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-jdk14-build + params: { state: "success", commit: "git-repo" } +- name: jdk15-build + serial: true + public: true + plan: + - get: spring-framework-jdk15-ci-image + - get: git-repo + trigger: true + - put: repo-status-jdk15-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-jdk15-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-jdk15-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] JDK 15 Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-jdk15-build + params: { state: "success", commit: "git-repo" } +groups: +- name: "Build" + jobs: ["build", "jdk11-build", "jdk14-build", "jdk15-build"] +- name: "CI Images" + jobs: ["build-spring-framework-ci-images"] diff --git a/ci/scripts/build-project.sh b/ci/scripts/build-project.sh new file mode 100755 index 00000000000..3844d1a3ddb --- /dev/null +++ b/ci/scripts/build-project.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +./gradlew -Dorg.gradle.internal.launcher.welcomeMessageEnabled=false --no-daemon --max-workers=4 -PdeploymentRepository=${repository} build publishAllPublicationsToDeploymentRepository +popd > /dev/null diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh new file mode 100644 index 00000000000..1accaa61673 --- /dev/null +++ b/ci/scripts/common.sh @@ -0,0 +1,2 @@ +source /opt/concourse-java.sh +setup_symlinks \ No newline at end of file diff --git a/ci/tasks/build-project.yml b/ci/tasks/build-project.yml new file mode 100644 index 00000000000..d9379fd803a --- /dev/null +++ b/ci/tasks/build-project.yml @@ -0,0 +1,21 @@ +--- +platform: linux +inputs: +- name: git-repo +outputs: +- name: distribution-repository +caches: +- path: gradle +params: + BRANCH: + CI: true + GRADLE_ENTERPRISE_ACCESS_KEY: + GRADLE_ENTERPRISE_CACHE_USERNAME: + GRADLE_ENTERPRISE_CACHE_PASSWORD: + GRADLE_ENTERPRISE_URL: https://ge.spring.io +run: + path: bash + args: + - -ec + - | + ${PWD}/git-repo/ci/scripts/build-project.sh