Jenkins pipelines - GitHub PR verification and SSH ID fix (#1189)
This commit contains two parts:
New Jenkins pipeline which will be used by GitHub pull request verification job as a replacement for Travis-CI (.org)
Fixes in current Jenkins pipelines to reflect some changes made by Eclipse.org stuff. Without these fixes Jenkins jobs should throw errors like "java.lang.RuntimeException: [ssh-agent] Could not find a suitable ssh-agent provider".
Signed-off-by: Radek Felcman <radek.felcman@oracle.com>diff --git a/etc/jenkins/build.groovy b/etc/jenkins/build.groovy
index 30426f6..3fa8d16 100644
--- a/etc/jenkins/build.groovy
+++ b/etc/jenkins/build.groovy
@@ -13,7 +13,6 @@
// Job input parameters (passed from Properties Content field from Jenkins job):
// GIT_REPOSITORY_URL - Git repository location (URL)
// GIT_BRANCH - Git branch
-// SSH_CREDENTIALS_ID - SSH credentials is used to access Git repository at the GitHub
// BUILD_RESULTS_TARGET_DIR - Location in the projects-storage.eclipse.org server for nightly builds (jar files and test results)
// CONTINUOUS_BUILD - false - full nightly build with LRG and server tests and nightly build publish
// true - continuous build with SRG tests without publishing nightly build results
@@ -109,11 +108,9 @@
steps {
container('el-build') {
git branch: '${GIT_BRANCH}', url: '${GIT_REPOSITORY_URL}'
- sshagent(['SSH_CREDENTIALS_ID']) {
- sh """
- etc/jenkins/init.sh
- """
- }
+ sh """
+ etc/jenkins/init.sh
+ """
withCredentials([file(credentialsId: 'secret-subkeys.asc', variable: 'KEYRING')]) {
sh label: '', script: '''
gpg --batch --import "${KEYRING}"
@@ -129,10 +126,9 @@
stage('Build') {
steps {
container('el-build') {
- sshagent(['SSH_CREDENTIALS_ID']) {
- sh """
- etc/jenkins/build.sh
- """
+ sh """
+ etc/jenkins/build.sh
+ """
}
}
}
@@ -183,11 +179,9 @@
stage('Publish to snapshots') {
steps {
container('el-build') {
- sshagent([SSH_CREDENTIALS_ID]) {
- sh """
- etc/jenkins/publish_snapshots.sh
- """
- }
+ sh """
+ etc/jenkins/publish_snapshots.sh
+ """
}
}
}
diff --git a/etc/jenkins/build_asm.groovy b/etc/jenkins/build_asm.groovy
index 0d7919d..755cfce 100644
--- a/etc/jenkins/build_asm.groovy
+++ b/etc/jenkins/build_asm.groovy
@@ -53,13 +53,10 @@
// Perform release
stage('Build') {
steps {
- sshagent([SSH_CREDENTIALS_ID]) {
- sh '''
- etc/jenkins/build_asm.sh
- '''
- }
+ sh '''
+ etc/jenkins/build_asm.sh
+ '''
}
}
-
}
}
diff --git a/etc/jenkins/pr_verify.groovy b/etc/jenkins/pr_verify.groovy
new file mode 100644
index 0000000..0260c0a
--- /dev/null
+++ b/etc/jenkins/pr_verify.groovy
@@ -0,0 +1,258 @@
+//
+// Copyright (c) 2021 Oracle and/or its affiliates. All rights reserved.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0,
+// or the Eclipse Distribution License v. 1.0 which is available at
+// http://www.eclipse.org/org/documents/edl-v10.php.
+//
+// SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
+//
+
+// Job input parameters (passed from Properties Content field from Jenkins job):
+// GIT_REPOSITORY_URL - Git repository location (URL)
+// GIT_BRANCH - Git branch
+// SSH_CREDENTIALS_ID - SSH credentials is used to access Git repository at the GitHub
+
+
+pipeline {
+ agent {
+ kubernetes {
+ label 'el-master-agent-pod'
+ yaml """
+apiVersion: v1
+kind: Pod
+spec:
+
+ volumes:
+ - name: tools
+ persistentVolumeClaim:
+ claimName: tools-claim-jiro-eclipselink
+ - name: volume-known-hosts
+ configMap:
+ name: known-hosts
+ - name: settings-xml
+ secret:
+ secretName: m2-secret-dir
+ items:
+ - key: settings.xml
+ path: settings.xml
+ - name: toolchains-xml
+ configMap:
+ name: m2-dir
+ items:
+ - key: toolchains.xml
+ path: toolchains.xml
+ - name: settings-security-xml
+ secret:
+ secretName: m2-secret-dir
+ items:
+ - key: settings-security.xml
+ path: settings-security.xml
+ - name: m2-repo
+ emptyDir: {}
+
+ containers:
+ - name: jnlp
+ resources:
+ limits:
+ memory: "4Gi"
+ cpu: "2"
+ requests:
+ memory: "4Gi"
+ cpu: "1"
+ - name: el-build
+ resources:
+ limits:
+ memory: "12Gi"
+ cpu: "6"
+ requests:
+ memory: "12Gi"
+ cpu: "5.5"
+ image: tkraus/el-build:1.1.9
+ volumeMounts:
+ - name: tools
+ mountPath: /opt/tools
+ - name: volume-known-hosts
+ mountPath: /home/jenkins/.ssh
+ - name: settings-xml
+ mountPath: /home/jenkins/.m2/settings.xml
+ subPath: settings.xml
+ readOnly: true
+ - name: toolchains-xml
+ mountPath: /home/jenkins/.m2/toolchains.xml
+ subPath: toolchains.xml
+ readOnly: true
+ - name: settings-security-xml
+ mountPath: /home/jenkins/.m2/settings-security.xml
+ subPath: settings-security.xml
+ readOnly: true
+ - name: m2-repo
+ mountPath: /home/jenkins/.m2/repository
+ tty: true
+ command:
+ - cat
+"""
+ }
+ }
+ tools {
+ maven 'apache-maven-latest'
+ jdk 'adoptopenjdk-hotspot-latest'
+ }
+ stages {
+ // Initialize build environment
+ stage('Init') {
+ steps {
+ container('el-build') {
+ git branch: '${GIT_BRANCH}', url: '${GIT_REPOSITORY_URL}'
+ sh """
+ /opt/bin/mysql-start.sh
+ mkdir ~/.eclipselinktests
+ """
+ withCredentials([file(credentialsId: 'secret-subkeys.asc', variable: 'KEYRING')]) {
+ sh label: '', script: '''
+ gpg --batch --import "${KEYRING}"
+ for fpr in $(gpg --list-keys --with-colons | awk -F: \'/fpr:/ {print $10}\' | sort -u);
+ do
+ echo -e "5\\ny\\n" | gpg --batch --command-fd 0 --expert --edit-key $fpr trust;
+ done'''
+ }
+ }
+ }
+ }
+ // Build
+ stage('Build') {
+ steps {
+ container('el-build') {
+ sh """
+ echo '-[ EclipseLink Build ]-----------------------------------------------------------'
+ mvn -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -B clean install -pl '!:eclipselink,!:org.eclipse.persistence.bundles.other,!:org.eclipse.persistence.distribution.tests,!:p2site' -DskipTests
+ """
+ }
+ }
+ }
+ // LRG tests
+ stage('Run Tests') {
+ parallel {
+ stage('Core LRG') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn verify -pl :org.eclipse.persistence.core.test -P test-core-lrg,mysql
+ """
+ }
+ }
+ }
+ stage('MOXy LRG') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn test -pl :org.eclipse.persistence.moxy -P test-moxy-lrg
+ """
+ }
+ }
+ }
+ stage('NoSQL Tests') {
+ steps {
+ container('el-build') {
+ sh """
+ /opt/bin/mongo-start.sh
+ mvn verify -pl :org.eclipse.persistence.nosql -P mongodb
+ /opt/bin/mongo-stop.sh
+ """
+ }
+ }
+ }
+ }
+ }
+ //Not in parallel
+ stage('JPA LRG') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn verify -pl :org.eclipse.persistence.jpa.test -P test-jpa-lrg,mysql
+ """
+ }
+ }
+ }
+ stage('SDO LRG') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn verify -pl :org.eclipse.persistence.sdo -Ptest-sdo
+ """
+ }
+ }
+ }
+ stage('CORBA') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn verify -pl :org.eclipse.persistence.corba -P mysql
+ """
+ }
+ }
+ }
+ stage('JPA Modelgen, JPA JSE, WDF, JPARS, DBWS, DBWS Builder, Distribution') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn clean install -pl :eclipselink
+ mvn verify -pl :org.eclipse.persistence.jpa.modelgen.processor,:org.eclipse.persistence.jpa.jse.test,:org.eclipse.persistence.extension,:org.eclipse.persistence.jpa.jpql,:org.eclipse.persistence.jpa.wdf.test,:org.eclipse.persistence.jpars,:org.eclipse.persistence.dbws,:org.eclipse.persistence.dbws.builder,:eclipselink,:org.eclipse.persistence.distribution.tests -P mysql;
+ """
+ }
+ }
+ }
+ stage('Javadoc') {
+ steps {
+ container('el-build') {
+ sh """
+ mvn package -DskipTests -Poss-release
+ """
+ }
+ }
+ }
+ // LRG Server tests
+/*
+ stage('LRG Server tests (Wildfly)') {
+ steps {
+ container('el-build') {
+ sh """
+ etc/jenkins/test_server.sh
+ """
+ }
+ }
+ }
+*/
+ }
+ post {
+ always {
+ container('el-build') {
+ sh """
+ /opt/bin/mysql-stop.sh
+ """
+ }
+ script {
+ //Multiple Jenkins junit plugin calls due java.nio.channels.ClosedChannelException in new/cloud Eclipse.org build infrastructure if it's called once
+ //Retry is there to try (in case of crash) junit test upload again.
+ retryCount = 5
+ junitReportFiles = [
+ 'bundles/**/target/surefire-reports/*.xml,bundles/**/target/failsafe-reports/*.xml',
+ 'dbws/**/target/surefire-reports/*.xml,dbws/**/target/failsafe-reports/*.xml',
+ 'foundation/**/target/surefire-reports/*.xml,foundation/**/target/failsafe-reports/*.xml',
+ 'jpa/**/target/surefire-reports/*.xml,jpa/**/target/failsafe-reports/*.xml',
+ 'moxy/**/target/surefire-reports/*.xml,moxy/**/target/failsafe-reports/*.xml',
+ 'sdo/**/target/surefire-reports/*.xml,sdo/**/target/failsafe-reports/*.xml',
+ 'utils/**/target/surefire-reports/*.xml,utils/**/target/failsafe-reports/*.xml'
+ ]
+ for (item in junitReportFiles) {
+ echo 'Processing file: ' + item
+ retry(retryCount) {
+ junit allowEmptyResults: true, testResults: item
+ }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/etc/jenkins/release.groovy b/etc/jenkins/release.groovy
index 9e78fb1..59202cf 100644
--- a/etc/jenkins/release.groovy
+++ b/etc/jenkins/release.groovy
@@ -1,4 +1,4 @@
-// Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2021 Oracle and/or its affiliates. All rights reserved.
//
// This program and the accompanying materials are made available under the
// terms of the Eclipse Distribution License v. 1.0, which is available at
@@ -108,13 +108,11 @@
steps {
container('el-build') {
git branch: GIT_BRANCH_RELEASE, credentialsId: SSH_CREDENTIALS_ID, url: GIT_REPOSITORY_URL
- sshagent([SSH_CREDENTIALS_ID]) {
- sh """
- # Directory for JEE server binaries (WildFly, Glassfish)
- # Maven build automatically download and unpack them.
- mkdir ~/.eclipselinktests
- """
- }
+ sh """
+ # Directory for JEE server binaries (WildFly, Glassfish)
+ # Maven build automatically download and unpack them.
+ mkdir ~/.eclipselinktests
+ """
withCredentials([file(credentialsId: 'secret-subkeys.asc', variable: 'KEYRING')]) {
sh label: '', script: '''
gpg --batch --import "${KEYRING}"
@@ -137,11 +135,9 @@
steps {
container('el-build') {
git branch: GIT_BRANCH_RELEASE, credentialsId: SSH_CREDENTIALS_ID, url: GIT_REPOSITORY_URL
- sshagent([SSH_CREDENTIALS_ID]) {
- sh """
- etc/jenkins/release.sh "${RELEASE_VERSION}" "${NEXT_VERSION}" "${DRY_RUN}" "${OVERWRITE_GIT}" "${OVERWRITE_STAGING}"
- """
- }
+ sh """
+ etc/jenkins/release.sh "${RELEASE_VERSION}" "${NEXT_VERSION}" "${DRY_RUN}" "${OVERWRITE_GIT}" "${OVERWRITE_STAGING}"
+ """
}
}
}
diff --git a/etc/jenkins/release_asm.groovy b/etc/jenkins/release_asm.groovy
index 4d01155..e4acb52 100644
--- a/etc/jenkins/release_asm.groovy
+++ b/etc/jenkins/release_asm.groovy
@@ -141,11 +141,9 @@
steps {
container('el-build') {
git branch: GIT_BRANCH_RELEASE, credentialsId: SSH_CREDENTIALS_ID, url: GIT_REPOSITORY_URL
- sshagent([SSH_CREDENTIALS_ID]) {
- sh '''
- etc/jenkins/release_asm.sh "${ASM_VERSION}" "${NEXT_ASM_VERSION}" "${DRY_RUN}" "${OVERWRITE_GIT}" "${OVERWRITE_STAGING}"
- '''
- }
+ sh '''
+ etc/jenkins/release_asm.sh "${ASM_VERSION}" "${NEXT_ASM_VERSION}" "${DRY_RUN}" "${OVERWRITE_GIT}" "${OVERWRITE_STAGING}"
+ '''
}
}
}
diff --git a/etc/jenkins/tck.groovy b/etc/jenkins/tck.groovy
index 071daf6..00f33ab 100644
--- a/etc/jenkins/tck.groovy
+++ b/etc/jenkins/tck.groovy
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2021 Oracle and/or its affiliates. All rights reserved.
//
// This program and the accompanying materials are made available under the
// terms of the Eclipse Public License v. 2.0 which is available at
