MacOS tests (#3887) (9a9c3505) · Commits · aflab / astrogeology / ISIS3

Jenkinsfile

+103 −197

Original line number	Diff line number	Diff line
		// vim: ft=groovy

		// Helpers for setting commit status
		def getRepoUrl() {
		return sh(script: "git config --get remote.origin.url", returnStdout: true).trim()
		}

		def getCommitSha() {
		return sh(script: "git rev-parse HEAD", returnStdout: true).trim()
		}


		def setGitHubBuildStatus(status) {
		def repoUrl = getRepoUrl()
		def commitSha = getCommitSha()

		step([
		$class: 'GitHubCommitStatusSetter',
		reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
		commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
		errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
		statusResultSource: [
		$class: 'ConditionalStatusResultSource',
		results: [
		[$class: 'BetterThanOrEqualBuildResult', result: 'SUCCESS', state: 'SUCCESS', message: status],
		[$class: 'BetterThanOrEqualBuildResult', result: 'FAILURE', state: 'FAILURE', message: status],
		[$class: 'AnyBuildResult', state: 'FAILURE', message: 'Loophole']
		]
		]
		])
		}

		pipeline {
		agent any
		stages {
		stage("CI") {
		steps {
		script {
		def groovy_utilities = load "${pwd()}/groovy_utilities.groovy"

		def NUM_CORES = 8
		def errors = []
		def labels = ['CentOS', 'Fedora', 'Ubuntu', 'Mac'] // labels for Jenkins node types we will build on
		def builders = [:]
		def nodes = [:]

		println("Using Labels: " + labels)
		for (lbl in labels) {
		def label = lbl
		def envFile = (label == "CentOS") ? "environment_gcc4.yml" : "environment.yml"

		for (x in labels) {
		def label = x
		def lower_label = label.toLowerCase()

		builders[lower_label] = {
		node(lower_label) {

		def build_ok = true
		def condaPath = ""
		nodes[label] = {
		stage(label) {
		isisNode(label) {
		def isisEnv = [
		"ISISDATA=/isisData/isis_data",
		"ISISTESTDATA=/isisData/isis_testData",
		@@ -67,150 +26,97 @@ pipeline {
		"-DCMAKE_BUILD_TYPE=RELEASE"
		]

		stage (label) {
		sh 'git config --global http.sslVerify false'
		checkout scm
		isisEnv.add("ISISROOT=${pwd()}/build")
		cmakeFlags.add("-DCMAKE_INSTALL_PREFIX=${pwd()}/install")
		def stageStatus = "Checking out ISIS on ${label}"

		env.STAGE_STATUS = "Creating conda environment"

		if (lower_label == "mac") {
		condaPath = "/tmp/macbuilds/" + sh(script: '{ date "+%m/%d/%y\|%H:%M:%S:%m"; echo $WORKSPACE; } \| md5 \| tr -d "\n";', returnStdout: true)
		// Checkout
		checkout scm

		sh """
		mkdir -p /tmp/macbuilds/
		curl -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
		bash miniconda.sh -b -p ${condaPath}
		condaEnv("isis3") {
		// Environment
		loginShell """
		conda install -c conda-forge python=3
		conda env update -f ${envFile} --prune
		mkdir build install
		"""
		} else {
		condaPath = "/home/jenkins/.conda/"
		}

		isisEnv.add("PATH=${pwd()}/install/bin:$condaPath/envs/isis/bin:$condaPath/bin:${env.PATH}")
		def osFailed = false
		isisEnv.add("ISISROOT=${pwd()}/build")
		isisEnv.add("PATH=${pwd()}/install/bin:${env.PATH}")
		cmakeFlags.add("-DCMAKE_INSTALL_PREFIX=${pwd()}/install")

		withEnv(isisEnv) {
		sh 'printenv'
		println("Anaconda Path: " + condaPath)

		sh """
		# Use the conda cache running on the Jenkins host
		conda config --set always_yes True
		conda config --set ssl_verify false
		conda create -c conda-forge --prefix ${condaPath}/envs/isis python=3
		"""

		if (lower_label == "centos") {
		sh "conda env update -p ${condaPath}/envs/isis -f environment_gcc4.yml --prune"
		} else {
		sh """
		conda config --show channels
		conda env update -p ${condaPath}/envs/isis -f environment.yml --prune
		"""
		}
		dir("${env.ISISROOT}") {
		dir(env.ISISROOT) {
		// Build
		stageStatus = "Building ISIS on ${label}"
		try {
		env.STAGE_STATUS = "Building ISIS on ${label}"
		sh """
		source activate ${condaPath}/envs/isis
		echo `ls ../`
		echo `pwd`
		conda list
		loginShell """
		cmake -GNinja ${cmakeFlags.join(' ')} ../isis
		ninja -j4 install
		ninja -j${NUM_CORES} install
		"""
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		} catch(e) {
		// Die right here
		error stageStatus
		}

		if (build_ok) {
		// Unit tests
		stageStatus = "Running unit tests on ${label}"
		try {
		dir("${env.ISISROOT}") {
		env.STAGE_STATUS = "Running unit tests on ${label}"
		sh """
		source activate ${condaPath}/envs/isis
		ctest -R _unit_ -j4 -VV
		"""
		}
		loginShell "ctest -R _unit_ -j${NUM_CORES} -VV"
		} catch(e) {
		errors.add(stageStatus)
		osFailed = true
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		echo e.toString()
		}
		sh 'source deactivate'

		// App tests
		stageStatus = "Running app tests on ${label}"
		try {
		env.STAGE_STATUS = "Running app tests on ${label}"
		sh """
		source activate ${condaPath}/envs/isis
		export PATH="${condaPath}bin/:$PATH"
		echo $PATH
		ctest -R _app_ -j4 -VV
		"""
		loginShell "ctest -R _app_ -j${NUM_CORES} -VV"
		} catch(e) {
		errors.add(stageStatus)
		osFailed = true
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'

		try {
		sh """
		source activate ${condaPath}/envs/isis
		ctest -R _module_ -j4 -VV
		"""
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		loginShell "ctest -R _module_ -j${NUM_CORES} -VV"
		} catch(e) {
		errors.add(stageStatus)
		osFailed = true
		}
		sh 'source deactivate'

		// Gtests
		stageStatus = "Running gtests on ${label}"
		try {
		env.STAGE_STATUS = "Running gtests on ${label}"
		sh """
		source activate ${condaPath}/envs/isis
		ctest -R "." -E "(_app_\|_unit_\|_module_)" -j4 -VV
		"""
		}
		catch(e) {
		build_ok = false
		errors.add(env.STAGE_STATUS)
		println e.toString()
		}
		sh 'source deactivate'
		}
		loginShell "ctest -R '.' -E '(_app_\|_unit_\|_module_)' -j${NUM_CORES} -VV"
		} catch(e) {
		errors.add(stageStatus)
		osFailed = true
		}

		if (lower_label == "mac") {
		sh "rm -rf ${condaPath}"
		}

		if(build_ok) {
		currentBuild.result = "SUCCESS"
		}
		else {
		currentBuild.result = "FAILURE"
		def comment = "Failed during:\n"
		errors.each {
		comment += "- ${it}\n"
		if (osFailed) {
		error "Failed on ${label}"
		}
		setGitHubBuildStatus(comment)
		}
		}
		}
		}
		}
		}
		parallel builders
		}
		}

		// Run the builds in parallel
		node {
		try {
		parallel nodes

		} catch(e) {
		// Report result to GitHub
		currentBuild.result = "FAILURE"

		def comment = "Failed during:\n"
		errors.each {
		comment += "- ${it}\n"
		}

		setGithubStatus(comment)
		}
		}

docker/centos.dockerfile

0 → 100644

+30 −0

Original line number	Diff line number	Diff line
		# vim: set ft=dockerfile:

		FROM usgsastro/centos:latest

		SHELL ["/bin/bash", "-lc"]

		RUN yum install -y \
		gcc \
		gcc-c++ \
		git \
		mesa-libGL-devel \
		&& \
		yum clean all

		RUN curl -sLo /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
		chmod +x /tmp/miniconda.sh && \
		/tmp/miniconda.sh -b -p /opt/conda && \
		rm /tmp/miniconda.sh && \
		echo '. /opt/conda/etc/profile.d/conda.sh' >> /etc/bashrc

		RUN git config --system http.sslCAInfo /etc/ssl/certs/ca-bundle.crt && \
		echo 'ssl_verify: /etc/ssl/certs/ca-bundle.crt' > /opt/conda/.condarc && \
		echo 'always_yes: true' >> /opt/conda/.condarc && \
		conda update conda

		RUN useradd -m conda
		USER conda
		WORKDIR /home/conda

		CMD ["/bin/bash", "-l"]

docker/fedora.dockerfile

0 → 100644

+32 −0

Original line number	Diff line number	Diff line
		# vim: set ft=dockerfile:

		FROM usgsastro/fedora:latest

		SHELL ["/bin/bash", "-lc"]

		RUN dnf makecache && \
		dnf install -y \
		gcc \
		gcc-c++ \
		git \
		libnsl \
		mesa-libGL-devel \
		&& \
		dnf clean all

		RUN curl -sLo /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
		chmod +x /tmp/miniconda.sh && \
		/tmp/miniconda.sh -b -p /opt/conda && \
		rm /tmp/miniconda.sh && \
		echo '. /opt/conda/etc/profile.d/conda.sh' >> /etc/bashrc

		RUN git config --system http.sslCAInfo /etc/ssl/certs/ca-bundle.crt && \
		echo 'ssl_verify: /etc/ssl/certs/ca-bundle.crt' > /opt/conda/.condarc && \
		echo 'always_yes: true' >> /opt/conda/.condarc && \
		conda update conda

		RUN useradd -m conda
		USER conda
		WORKDIR /home/conda

		CMD ["/bin/bash", "-l"]

docker/ubuntu.dockerfile

0 → 100644

+33 −0

Original line number	Diff line number	Diff line
		# vim: set ft=dockerfile:

		FROM usgsastro/ubuntu:latest

		SHELL ["/bin/bash", "-lc"]

		RUN apt-get update && \
		apt-get install -y \
		curl \
		gcc \
		g++ \
		git \
		libgl1-mesa-dev \
		&& \
		rm -rf /var/cache/apt/*

		RUN curl -sLo /tmp/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
		chmod +x /tmp/miniconda.sh && \
		/tmp/miniconda.sh -b -p /opt/conda && \
		rm /tmp/miniconda.sh && \
		ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
		echo '. /opt/conda/etc/profile.d/conda.sh' >> /etc/bash.bashrc

		RUN git config --system http.sslCAInfo /etc/ssl/certs/ca-certificates.crt && \
		echo 'ssl_verify: /etc/ssl/certs/ca-certificates.crt' > /opt/conda/.condarc && \
		echo 'always_yes: true' >> /opt/conda/.condarc && \
		conda update conda

		RUN useradd -m conda
		USER conda
		WORKDIR /home/conda

		CMD ["/bin/bash", "-l"]

groovy_utilities.groovy

deleted100644 → 0

+0 −32

Original line number	Diff line number	Diff line
		// vim: ft=groovy

		// Helpers for setting commit status
		def getRepoUrl() {
		return sh(script: "git config --get remote.origin.url", returnStdout: true).trim()
		}

		def getCommitSha() {
		return sh(script: "git rev-parse HEAD", returnStdout: true).trim()
		}

		def setGitHubBuildStatus(status) {
		def repoUrl = getRepoUrl()
		def commitSha = getCommitSha()

		step([
		$class: 'GitHubCommitStatusSetter',
		reposSource: [$class: "ManuallyEnteredRepositorySource", url: repoUrl],
		commitShaSource: [$class: "ManuallyEnteredShaSource", sha: commitSha],
		errorHandlers: [[$class: 'ShallowAnyErrorHandler']],
		statusResultSource: [
		$class: 'ConditionalStatusResultSource',
		results: [
		[$class: 'BetterThanOrEqualBuildResult', result: 'SUCCESS', state: 'SUCCESS', message: status],
		[$class: 'BetterThanOrEqualBuildResult', result: 'FAILURE', state: 'FAILURE', message: status],
		[$class: 'AnyBuildResult', state: 'FAILURE', message: 'Loophole']
		]
		]
		])
		}

		return this