diff --git a/README.md b/README.md index 32aa262d732..93b3c486667 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Spring Framework [![Build Status](https://build.spring.io/plugins/servlet/wittified/build-status/SPR-PUBM)](https://build.spring.io/browse/SPR) +# Spring Framework [![Build Status](https://ci.spring.io/api/v1/teams/spring-framework/pipelines/spring-framework-5.3.x/jobs/build/badge)](https://ci.spring.io/teams/spring-framework/pipelines/spring-framework-5.3.x?groups=Build") This is the home of the Spring Framework: the foundation for all [Spring projects](https://spring.io/projects). Collectively the Spring Framework and the family of Spring projects are often referred to simply as "Spring". diff --git a/ci/README.adoc b/ci/README.adoc new file mode 100644 index 00000000000..c2c154acca9 --- /dev/null +++ b/ci/README.adoc @@ -0,0 +1,51 @@ +== Spring Framework Concourse pipeline + +The Spring Framework is using https://concourse-ci.org/[Concourse] for its CI build and other automated tasks. +The Spring team has a dedicated Concourse instance available at https://ci.spring.io. + +=== Setting up your development environment + +If you're part of the Spring Framework project on GitHub, you can get access to CI management features. +First, you need to go to https://ci.spring.io and install the client CLI for your platform (see bottom right of the screen). + +You can then login with the instance using: + +[source] +---- +$ fly -t spring login -n spring-framework -c https://ci.spring.io +---- + +Once logged in, you should get something like: + +[source] +---- +$ fly ts +name url team expiry +spring https://ci.spring.io spring-framework Wed, 25 Mar 2020 17:45:26 UTC +---- + +=== Pipeline configuration and structure +The build pipelines are described in `pipeline.yml` file. +This file is listing Concourse resources, i.e. build inputs and outputs such as container images, artifact repositories, source repositories, notification services, etc. +It also describes jobs (a job is a sequence of inputs, tasks and outputs); jobs are organized by groups. + +The `pipeline.yml` definition contains `((parameters))` which are loaded from the `parameters.yml` file or from our https://docs.cloudfoundry.org/credhub/[credhub instance]. + +You'll find in this folder the following resources: +* `pipeline.yml` the build pipeline +* `parameters.yml` the build parameters used for the pipeline +* `images/` holds the container images definitions used in this pipeline +* `scripts/` holds the build scripts that ship within the CI container images +* `tasks` contains the task definitions used in the main `pipeline.yml` + +=== Updating the build pipeline + +Updating files on the repository is not enough to update the build pipeline, as changes need to be applied. +The pipeline can be deployed using the following command: + +[source] +---- +$ fly -t spring set-pipeline -p spring-framework-5.3.x -c ci/pipeline.yml -l ci/parameters.yml +---- + +NOTE: This assumes that you have credhub integration configured with the appropriate secrets. \ No newline at end of file diff --git a/ci/images/README.adoc b/ci/images/README.adoc new file mode 100644 index 00000000000..6da9addd9ca --- /dev/null +++ b/ci/images/README.adoc @@ -0,0 +1,21 @@ +== CI Images + +These images are used by CI to run the actual builds. + +To build the image locally run the following from this directory: + +---- +$ docker build --no-cache -f /Dockerfile . +---- + +For example + +---- +$ docker build --no-cache -f spring-framework-ci-image/Dockerfile . +---- + +To test run: + +---- +$ docker run -it --entrypoint /bin/bash +---- diff --git a/ci/images/get-jdk-url.sh b/ci/images/get-jdk-url.sh new file mode 100755 index 00000000000..d26fd413cc7 --- /dev/null +++ b/ci/images/get-jdk-url.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +case "$1" in + java8) + echo "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u252b09.tar.gz" + ;; + java11) + echo "https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.7%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.7_10.tar.gz" + ;; + java14) + echo "https://github.com/AdoptOpenJDK/openjdk14-binaries/releases/download/jdk-14.0.1%2B7/OpenJDK14U-jdk_x64_linux_hotspot_14.0.1_7.tar.gz" + ;; + java15) + echo "https://download.java.net/java/early_access/jdk15/20/GPL/openjdk-15-ea+20_linux-x64_bin.tar.gz" + ;; + *) + echo $"Unknown java version" + exit 1 +esac diff --git a/ci/images/setup.sh b/ci/images/setup.sh new file mode 100755 index 00000000000..b9eab7adba5 --- /dev/null +++ b/ci/images/setup.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -ex + +########################################################### +# UTILS +########################################################### + +apt-get update +apt-get install --no-install-recommends -y ca-certificates net-tools libxml2-utils git curl libudev1 libxml2-utils iptables iproute2 jq fontconfig +rm -rf /var/lib/apt/lists/* + +curl https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.2/concourse-java.sh > /opt/concourse-java.sh + + +########################################################### +# JAVA +########################################################### +JDK_URL=$( ./get-jdk-url.sh $1 ) + +mkdir -p /opt/openjdk +cd /opt/openjdk +curl -L ${JDK_URL} | tar zx --strip-components=1 +test -f /opt/openjdk/bin/java +test -f /opt/openjdk/bin/javac + +########################################################### +# GRADLE ENTERPRISE +########################################################### +cd / +mkdir ~/.gradle +echo 'systemProp.user.name=concourse' > ~/.gradle/gradle.properties \ No newline at end of file diff --git a/ci/images/spring-framework-ci-image/Dockerfile b/ci/images/spring-framework-ci-image/Dockerfile new file mode 100644 index 00000000000..cfd64e44fea --- /dev/null +++ b/ci/images/spring-framework-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java8 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/spring-framework-jdk11-ci-image/Dockerfile b/ci/images/spring-framework-jdk11-ci-image/Dockerfile new file mode 100644 index 00000000000..df007ebf5d9 --- /dev/null +++ b/ci/images/spring-framework-jdk11-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java11 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/spring-framework-jdk14-ci-image/Dockerfile b/ci/images/spring-framework-jdk14-ci-image/Dockerfile new file mode 100644 index 00000000000..a1dab0b61eb --- /dev/null +++ b/ci/images/spring-framework-jdk14-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java14 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/spring-framework-jdk15-ci-image/Dockerfile b/ci/images/spring-framework-jdk15-ci-image/Dockerfile new file mode 100644 index 00000000000..ada77812c83 --- /dev/null +++ b/ci/images/spring-framework-jdk15-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:bionic-20200311 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java15 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/parameters.yml b/ci/parameters.yml new file mode 100644 index 00000000000..3e09f785ebf --- /dev/null +++ b/ci/parameters.yml @@ -0,0 +1,14 @@ +email-server: "smtp.svc.pivotal.io" +email-from: "ci@spring.io" +email-to: ["spring-framework-dev@pivotal.io"] +github-repo: "https://github.com/spring-projects/spring-framework.git" +github-repo-name: "spring-projects/spring-framework" +docker-hub-organization: "springci" +artifactory-server: "https://repo.spring.io" +branch: "master" +build-name: "spring-framework" +pipeline-name: "spring-framework" +concourse-url: "https://ci.spring.io" +bintray-subject: "spring" +bintray-repo: "jars" +task-timeout: 1h00m \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 00000000000..539a355077b --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,323 @@ +resource_types: +- name: artifactory-resource + type: docker-image + source: + repository: springio/artifactory-resource + tag: 0.0.12-SNAPSHOT +- name: github-status-resource + type: docker-image + source: + repository: dpb587/github-status-resource + tag: master +- name: email + type: docker-image + source: + repository: pcfseceng/email-resource + tag: 1.0.19 +- name: slack-notification + type: docker-image + source: + repository: cfcommunity/slack-notification-resource + tag: latest +resources: +- name: git-repo + type: git + icon: github-circle + source: + uri: ((github-repo)) + username: ((github-username)) + password: ((github-password)) + branch: ((branch)) +- name: ci-images-git-repo + type: git + icon: github-circle + source: + uri: ((github-repo)) + branch: ((branch)) + paths: ["ci/images/*"] +- name: spring-framework-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: spring-framework-jdk11-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-jdk11-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: spring-framework-jdk14-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-jdk14-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: spring-framework-jdk15-ci-image + type: docker-image + icon: docker + source: + repository: ((docker-hub-organization))/spring-framework-jdk15-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: 5.3.x +- name: artifactory-repo + type: artifactory-resource + icon: package-variant + source: + uri: ((artifactory-server)) + username: ((artifactory-username)) + password: ((artifactory-password)) + build_name: ((build-name)) +- name: repo-status-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: build +- name: repo-status-jdk11-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: jdk11-build +- name: repo-status-jdk14-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: jdk14-build +- name: repo-status-jdk15-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: jdk15-build +- name: send-an-email + type: email + source: + smtp: + host: ((email-server)) + port: "25" + anonymous: true + from: ((email-from)) + to: ((email-to)) +- name: slack-alert + type: slack-notification + icon: slack + source: + url: ((slack-webhook-url)) +jobs: +- name: build-spring-framework-ci-images + plan: + - get: ci-images-git-repo + trigger: true + - in_parallel: + - put: spring-framework-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-ci-image/Dockerfile + - put: spring-framework-jdk11-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-jdk11-ci-image/Dockerfile + - put: spring-framework-jdk14-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-jdk14-ci-image/Dockerfile + - put: spring-framework-jdk15-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-framework-jdk15-ci-image/Dockerfile +- name: build + serial: true + public: true + plan: + - get: spring-framework-ci-image + - get: git-repo + trigger: true + - put: repo-status-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-build + params: { state: "success", commit: "git-repo" } + - put: artifactory-repo + params: &artifactory-params + repo: libs-snapshot-local + folder: distribution-repository + build_uri: "https://ci.spring.io/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}" + build_number: "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" + disable_checksum_uploads: true + threads: 8 + artifact_set: + - include: + - "/**/spring-framework-*.zip" + properties: + "zip.name": "spring-framework" + "zip.displayname": "Spring Framework" + "zip.deployed": "false" + - include: + - "/**/spring-framework-*-docs.zip" + properties: + "zip.type": "docs" + - include: + - "/**/spring-framework-*-dist.zip" + properties: + "zip.type": "dist" + - include: + - "/**/spring-framework-*-schema.zip" + properties: + "zip.type": "schema" + get_params: + threads: 8 +- name: jdk11-build + serial: true + public: true + plan: + - get: spring-framework-jdk11-ci-image + - get: git-repo + trigger: true + - put: repo-status-jdk11-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-jdk11-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-jdk11-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] JDK 11 Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-jdk11-build + params: { state: "success", commit: "git-repo" } +- name: jdk14-build + serial: true + public: true + plan: + - get: spring-framework-jdk14-ci-image + - get: git-repo + trigger: true + - put: repo-status-jdk14-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-jdk14-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-jdk14-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] JDK 14 Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-jdk14-build + params: { state: "success", commit: "git-repo" } +- name: jdk15-build + serial: true + public: true + plan: + - get: spring-framework-jdk15-ci-image + - get: git-repo + trigger: true + - put: repo-status-jdk15-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-framework-jdk15-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + on_failure: + do: + - put: repo-status-jdk15-build + params: { state: "failure", commit: "git-repo" } + - put: send-an-email + params: + subject_text: "[spring-framework-ci] JDK 15 Build ${BUILD_ID} failed" + body_text: "See build failure: ${ATC_EXTERNAL_URL}" + - put: slack-alert + params: + text: ":concourse-failed: " + silent: true + icon_emoji: ":concourse:" + username: concourse-ci + - put: repo-status-jdk15-build + params: { state: "success", commit: "git-repo" } +groups: +- name: "Build" + jobs: ["build", "jdk11-build", "jdk14-build", "jdk15-build"] +- name: "CI Images" + jobs: ["build-spring-framework-ci-images"] diff --git a/ci/scripts/build-project.sh b/ci/scripts/build-project.sh new file mode 100755 index 00000000000..3844d1a3ddb --- /dev/null +++ b/ci/scripts/build-project.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +./gradlew -Dorg.gradle.internal.launcher.welcomeMessageEnabled=false --no-daemon --max-workers=4 -PdeploymentRepository=${repository} build publishAllPublicationsToDeploymentRepository +popd > /dev/null diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh new file mode 100644 index 00000000000..1accaa61673 --- /dev/null +++ b/ci/scripts/common.sh @@ -0,0 +1,2 @@ +source /opt/concourse-java.sh +setup_symlinks \ No newline at end of file diff --git a/ci/tasks/build-project.yml b/ci/tasks/build-project.yml new file mode 100644 index 00000000000..d9379fd803a --- /dev/null +++ b/ci/tasks/build-project.yml @@ -0,0 +1,21 @@ +--- +platform: linux +inputs: +- name: git-repo +outputs: +- name: distribution-repository +caches: +- path: gradle +params: + BRANCH: + CI: true + GRADLE_ENTERPRISE_ACCESS_KEY: + GRADLE_ENTERPRISE_CACHE_USERNAME: + GRADLE_ENTERPRISE_CACHE_PASSWORD: + GRADLE_ENTERPRISE_URL: https://ge.spring.io +run: + path: bash + args: + - -ec + - | + ${PWD}/git-repo/ci/scripts/build-project.sh