Merging build.groovy into Jenkinsfile (#3620) (c40082af) · Commits · aflab / astrogeology / ISIS3

Jenkinsfile

+200 −199

Original line number	Diff line number	Diff line
		// vim: ft=groovy

		pipeline
		{
		agent any
		// Helpers for setting commit status
		def getRepoUrl() {
		return sh(script: "git config --get remote.origin.url", returnStdout: true).trim()
		}

		def getCommitSha() {
		return sh(script: "git rev-parse HEAD", returnStdout: true).trim()
		}


		def setGitHubBuildStatus(status) {
		def repoUrl = getRepoUrl()
		def commitSha = getCommitSha()

		step([
		$class: 'GitHubCommitStatusSetter',
		reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
		commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
		errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
		statusResultSource: [
		$class: 'ConditionalStatusResultSource',
		results: [
		[$class: 'BetterThanOrEqualBuildResult', result: 'SUCCESS', state: 'SUCCESS', message: status],
		[$class: 'BetterThanOrEqualBuildResult', result: 'FAILURE', state: 'FAILURE', message: status],
		[$class: 'AnyBuildResult', state: 'FAILURE', message: 'Loophole']
		]
		]
		])
		}

		pipeline {
		agent any
		stages {
		stage("CI") {
		steps {
		script {
		def groovy_utilities = load "${pwd()}/groovy_utilities.groovy"

		def isisDataPath = '/isisData/data'
		def errors = []
		def labels = ['CentOS', 'Fedora', 'Ubuntu', 'Mac'] // labels for Jenkins node types we will build on
		def builders = [:]

		def isisMgrScripts = '/isisData/data/isis3mgr_scripts'
		println("Using Labels: " + labels)

		def isisTestDataPath = "/isisData/testData"
		for (x in labels) {
		def label = x
		def lower_label = label.toLowerCase()

		def kakaduIncDir = "/isisData/kakadu"
		builders[lower_label] = {
		node(lower_label) {

		def build_ok = true
		def condaPath = ""
		def isisEnv = [
		"ISIS3DATA=${isisDataPath}",
		"ISIS3TESTDATA=${isisTestDataPath}",
		"ISIS3MGRSCRIPTS=${isisMgrScripts}"
		"ISIS3DATA=/isisData/data",
		"ISIS3TESTDATA=/isisData/testData",
		"ISIS3MGRSCRIPTS=/isisData/data/isis3mgr_scripts",
		]

		def cmakeFlags = [
		"-DJP2KFLAG=ON",
		"-DKAKADU_INCLUDE_DIR=${kakaduIncDir}",
		"-DKAKADU_INCLUDE_DIR=/isisData/kakadu",
		"-Dpybindings=OFF",
		"-DCMAKE_BUILD_TYPE=RELEASE"
		]

		def build_ok = true
		def errors = []

		def labels = ['CentOS', 'Fedora', 'Ubuntu'] // labels for Jenkins node types we will build on
		def builders = [:]

		for (x in labels) {
		def label = x // Need to bind the label variable before the closure - can't do 'for (label in labels)'
		def lower_label = x.toLowerCase()

		// Create a map to pass in to the 'parallel' step so we can fire all the builds at once
		builders[lower_label] = {
		node(lower_label) {
		stage ("${label}") {

		env.STAGE_STATUS = "Checking out ISIS"
		stage (label) {
		sh 'git config --global http.sslVerify false'
		checkout scm
		isisEnv.add("ISISROOT=${pwd()}/build")
		cmakeFlags.add("-DCMAKE_INSTALL_PREFIX=${pwd()}/install")

		env.STAGE_STATUS = "Creating conda environment"
		sh '''

		if (lower_label == "mac") {
		condaPath = "/tmp/" + sh(script: '{ date "+%m/%d/%y\|%H:%M:%S:%m"; echo $WORKSPACE; } \| md5 \| tr -d "\n";', returnStdout: true)

		sh """
		curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
		bash miniconda.sh -b -p ${condaPath}
		"""
		} else {
		condaPrefix = "/home/jenkins/.conda/envs/isis/"
		condaPath = "/home/jenkins/.conda/"
		}

		isisEnv.add("PATH=${pwd()}/install/bin:$condaPath/envs/isis/bin:$condaPath/bin:${env.PATH}")

		withEnv(isisEnv) {
		sh 'printenv'
		println("Anaconda Path: " + condaPath)

		sh """
		# Use the conda cache running on the Jenkins host
		conda config --set channel_alias http://dmz-jenkins.wr.usgs.gov
		conda config --set always_yes True
		conda config --set ssl_verify false
		conda create -n isis python=3
		'''
		conda create -c conda-forge --prefix ${condaPath}/envs/isis python=3
		"""

		if (lower_label == "centos") {
		sh 'conda env update -n isis -f environment_gcc4.yml --prune'
		sh "conda env update -p ${condaPath}/envs/isis -f environment_gcc4.yml --prune"
		} else {
		sh 'conda env update -n isis -f environment.yml --prune'
		sh """
		conda config --show channels
		conda env update -p ${condaPath}/envs/isis -f environment.yml --prune
		"""
		}

		withEnv(isisEnv) {
		dir("${env.ISISROOT}") {
		try {
		env.STAGE_STATUS = "Building ISIS on ${label}"
		sh """
		source activate isis
		source activate ${condaPath}/envs/isis
		echo `ls ../`
		echo `pwd`
		conda list
		cmake -GNinja ${cmakeFlags.join(' ')} ../isis
		ninja -j4 install
		python ../isis/scripts/isis3VarInit.py --data-dir ${env.ISIS3DATA} --test-dir ${env.ISIS3TESTDATA}
		"""
		}
		catch(e) {
		@@ -83,48 +124,30 @@ pipeline
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		}

		if (build_ok) {
		dir("${env.ISISROOT}") {
		try{
		env.STAGE_STATUS = "Running unit tests on ${env.OS}"
		dir("${env.ISISROOT}") {
		env.STAGE_STATUS = "Running unit tests on ${label}"
		sh """
		source activate isis
		echo $ISIS3TESTDATA
		echo $ISIS3DATA

		# environment variables
		export ISISROOT=${env.ISISROOT}
		export ISIS3TESTDATA="/isisData/testData"
		export ISIS3DATA="/isisData/data"
		export PATH=`pwd`/../install/bin:/home/jenkins/.conda/envs/isis/bin:$PATH

		source activate ${condaPath}/envs/isis
		ctest -R _unit_ -j4 -VV
		"""
		}
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		echo e.toString()
		}
		sh 'source deactivate'

		try{
		env.STAGE_STATUS = "Running app tests on ${env.OS}"
		env.STAGE_STATUS = "Running app tests on ${label}"
		sh """
		source activate isis
		echo $ISIS3TESTDATA
		echo $ISIS3DATA
		source activate ${condaPath}/envs/isis
		echo $PATH

		# environment variables
		export ISISROOT=${env.ISISROOT}
		export ISIS3TESTDATA="/isisData/testData"
		export ISIS3DATA='/isisData/data'
		export PATH=`pwd`/../install/bin:/home/jenkins/.conda/envs/isis/bin:$PATH

		ctest -R _app_ -j4 -VV
		source deactivate

		"""
		}
		catch(e) {
		@@ -132,24 +155,12 @@ pipeline
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'

		try{
		env.STAGE_STATUS = "Running module tests on ${env.OS}"
		sh """
		source activate isis
		echo $ISIS3TESTDATA
		echo $ISIS3DATA
		echo $PATH

		# environment variables
		export ISISROOT=${env.ISISROOT}
		export ISIS3TESTDATA="/isisData/testData"
		export ISIS3DATA='/isisData/data'
		export PATH=`pwd`/../install/bin:/home/jenkins/.conda/envs/isis/bin:$PATH

		source activate ${condaPath}/envs/isis
		ctest -R _module_ -j4 -VV
		source deactivate

		"""
		}
		catch(e) {
		@@ -157,24 +168,13 @@ pipeline
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'

		try{
		env.STAGE_STATUS = "Running gtests on ${env.OS}"
		env.STAGE_STATUS = "Running gtests on ${label}"
		sh """
		source activate isis
		echo $ISIS3TESTDATA
		echo $ISIS3DATA
		echo $PATH

		# environment variables
		export ISISROOT=${env.ISISROOT}
		export ISIS3TESTDATA="/isisData/testData"
		export ISIS3DATA='/isisData/data'
		export PATH=`pwd`/../install/bin:/home/jenkins/.conda/envs/isis/bin:$PATH

		source activate ${condaPath}/envs/isis
		ctest -R "." -E "(_app_\|_unit_\|_module_)" -j4 -VV
		source deactivate

		"""
		}
		catch(e) {
		@@ -182,6 +182,7 @@ pipeline
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'
		}
		}

		@@ -194,7 +195,7 @@ pipeline
		errors.each {
		comment += "- ${it}\n"
		}
		groovy_utilities.setGitHubBuildStatus(comment)
		setGitHubBuildStatus(comment)
		}
		}
		}

build.groovy

deleted100644 → 0

+0 −211

Original line number	Diff line number	Diff line
		// vim: ft=groovy

		def condaPath = ""

		def isisEnv = [
		"ISIS3DATA=/isisData/data",
		"ISIS3TESTDATA=/isisData/testData",
		"ISIS3MGRSCRIPTS=/isisData/data/isis3mgr_scripts",
		]

		def cmakeFlags = [
		"-DJP2KFLAG=ON",
		"-DKAKADU_INCLUDE_DIR=/isisData/kakadu",
		"-Dpybindings=OFF",
		"-DCMAKE_BUILD_TYPE=RELEASE"
		]

		def build_ok = true
		def errors = []

		// Helpers for setting commit status
		def getRepoUrl() {
		return sh(script: "git config --get remote.origin.url", returnStdout: true).trim()
		}

		def getCommitSha() {
		return sh(script: "git rev-parse HEAD", returnStdout: true).trim()
		}


		def setGitHubBuildStatus(status) {
		def repoUrl = getRepoUrl()
		def commitSha = getCommitSha()

		step([
		$class: 'GitHubCommitStatusSetter',
		reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
		commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
		errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
		statusResultSource: [
		$class: 'ConditionalStatusResultSource',
		results: [
		[$class: 'BetterThanOrEqualBuildResult', result: 'SUCCESS', state: 'SUCCESS', message: status],
		[$class: 'BetterThanOrEqualBuildResult', result: 'FAILURE', state: 'FAILURE', message: status],
		[$class: 'AnyBuildResult', state: 'FAILURE', message: 'Loophole']
		]
		]
		])
		}

		node("${env.OS.toLowerCase()}") {
		stage ("Checkout") {
		env.STAGE_STATUS = "Checking out ISIS"
		sh 'git config --global http.sslVerify false'
		checkout scm
		isisEnv.add("ISISROOT=${pwd()}/build")
		cmakeFlags.add("-DCMAKE_INSTALL_PREFIX=${pwd()}/install")
		}

		stage("Create environment") {

		env.STAGE_STATUS = "Creating conda environment"

		if (env.OS.toLowerCase() == "mac") {
		condaPath = "/tmp/" + sh(script: '{ date "+%m/%d/%y\|%H:%M:%S:%m"; echo $WORKSPACE; } \| md5 \| tr -d "\n";', returnStdout: true)

		sh """
		curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
		bash miniconda.sh -b -p ${condaPath}
		"""
		} else {
		condaPath = "/home/jenkins/.conda/"
		}

		isisEnv.add("PATH=${pwd()}/install/bin:$condaPath/envs/isis/bin:$condaPath/bin:${env.PATH}")

		withEnv(isisEnv) {

		println("Complete Environment:")
		sh 'printenv'
		println("Anaconda Path: " + condaPath)

		sh """
		# Use the conda cache running on the Jenkins host
		# conda config --set channel_alias http://dmz-jenkins.wr.usgs.gov
		conda config --set always_yes True
		conda config --set ssl_verify false
		conda update -n base -c defaults conda
		"""


		if (env.OS.toLowerCase() == "centos") {
		sh 'conda env create -n isis -f environment_gcc4.yml'
		} else {
		sh """
		conda config --show channels
		conda env create -n isis -f environment.yml
		"""
		}
		}
		}

		withEnv(isisEnv) {
		dir("${env.ISISROOT}") {
		try {
		stage ("Build") {
		env.STAGE_STATUS = "Building ISIS on ${env.OS}"
		sh """
		source activate ${condaPath}/envs/isis
		echo `ls ../`
		echo `pwd`
		conda list
		cmake -GNinja ${cmakeFlags.join(' ')} ../isis
		ninja -j4 install
		"""
		}
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}

		if (build_ok) {

		try{
		stage("UnitTests") {
		dir("${env.ISISROOT}") {
		env.STAGE_STATUS = "Running unit tests on ${env.OS}"
		sh """
		source activate ${condaPath}/envs/isis
		ctest -R _unit_ -j4 -VV
		"""

		}
		}
		}
		catch(e) {
		build_ok = false
		echo e.toString()
		}
		sh 'source deactivate'

		try{
		stage("AppTests") {
		env.STAGE_STATUS = "Running app tests on ${env.OS}"
		sh """
		source activate ${condaPath}/envs/isis
		echo $PATH
		ctest -R _app_ -j4 -VV
		"""
		}
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'

		try{
		stage("ModuleTests") {
		env.STAGE_STATUS = "Running module tests on ${env.OS}"
		sh """
		source activate ${condaPath}/envs/isis
		ctest -R _module_ -j4 -VV
		"""
		}
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'

		try{
		stage("GTests") {
		env.STAGE_STATUS = "Running gtests on ${env.OS}"
		sh """
		source activate ${condaPath}/envs/isis
		ctest -R "." -E "(_app_\|_unit_\|_module_)" -j4 -VV
		"""
		}
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'
		}
		}

		if(build_ok) {
		currentBuild.result = "SUCCESS"
		}
		else {
		currentBuild.result = "FAILURE"
		def comment = "Failed during:\n"
		errors.each {
		comment += "- ${it}\n"
		}
		setGitHubBuildStatus(comment)
		}
		}

		stage("Clean Up") {
		env.STAGE_STATUS = "Removing conda environment"
		}
		}

environment_gcc4.yml

+1 −1

Original line number	Diff line number	Diff line
		@@ -19,7 +19,7 @@ dependencies:
		- doxygen
		- eigen
		- embree
		- geos>=3.7,>3.8
		- geos=3.7.1
		- geotiff
		- gmm
		- gmp