CT-147: first step in moving csp-lmc-mid from docker-compose (e55ed873) · Commits · Gianluca Marotta / csp-lmc-old

.gitlab-ci.yml

+1 −1

Original line number	Diff line number	Diff line
		@@ -41,8 +41,8 @@ stages:
		- test_common
		- publish_common
		- build
		- test
		- linting
		- test
		- pages
		- publish
		- release

csp-lmc-mid/.gitlab-ci.yml

+58 −20

Original line number	Diff line number	Diff line
		@@ -52,7 +52,7 @@ build:csp-lmc-mid_image:
		- cd $BUILD_PATH
		- make build
		variables:
		BUILD_PATH: csp-lmc-mid/docker
		BUILD_PATH: csp-lmc-mid

		# test:csp-lmc-mid:
		# stage: test
		@@ -76,38 +76,76 @@ build:csp-lmc-mid_image:
		#

		.test:
		image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
		before_script:
		- docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
		tags:
		- docker-executor
		stage: test
		artifacts:
		paths:
		- ./$BUILD_PATH/build/
		variables:
		BUILD_PATH: csp-lmc-mid/docker

		test:csp-lmc-mid:
		BUILD_PATH: csp-lmc-mid
		#
		#unit:csp-lmc-mid:
		# extends: .test
		# stage: test
		# script:
		# - cd $BUILD_PATH
		# - pwd
		# - python3 -m pip install -r requirements-tst.txt
		# - python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt
		# - python3 setup.py test --addopts='-k-integration' \|tee setup_py_test.stdout
		# - pwd
		# - mkdir -p build/reports
		# - ls -alR
		# - mv setup_py_test.stdout build/csp-lmc-mid-setup-test.stdout
		# - mv htmlcov build/csp-lmc-mid_htmlcov
		# - mv coverage.xml build

		integration:csp-lmc-mid:
		tags:
		- auto-k8sv2
		extends: .test
		stage: test
		image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
		script:
		- curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.2/bin/linux/amd64/kubectl
		- chmod +x ./kubectl
		- sudo mv ./kubectl /usr/local/bin/kubectl
		- curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
		- chmod 700 get_helm.sh
		- ./get_helm.sh
		- kubectl get pod --all-namespaces
		- kubectl get namespaces
		- cd $BUILD_PATH
		- export KUBE_NAMESPACE="ci-$CI_COMMIT_SHORT_SHA"
		- make k8s_test
		after_script:
		- export KUBE_NAMESPACE="ci-$CI_COMMIT_SHORT_SHA"
		- cd $BUILD_PATH
		- make test
		- pwd
		- make delete
		- make delete_namespace

		# linting stage
		.linting:
		extends: .test
		#
		linting:csp-lmc-mid:
		image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
		stage: linting
		tags:
		- docker-executor
		artifacts:
		paths:
		- ./$BUILD_PATH/build/
		dependencies: []
		script:
		- apt-get -y update
		- apt-get install -y python3-pip python3-setuptools python3-wheel --no-install-recommends
		- python3 -m pip install pylint2junit
		- cd $BUILD_PATH
		- make lint

		linting:csp-lmc-mid:
		extends: .linting
		dependencies: []
		- python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt
		- mkdir -p build/reports
		- pylint --output-format=parseable csp_lmc_mid \| tee ./build/csp_lmc_mid-code-analysis.stdout
		- pylint --output-format=pylint2junit.JunitReporter csp_lmc_mid > ./build/reports/csp-lmc-mid-linting.xml
		- ls -alR
		variables:
		BUILD_PATH: csp-lmc-mid

		#
		# Publish csp-lmc-mid python package
		@@ -122,7 +160,7 @@ linting:csp-lmc-mid:
		tags:
		- docker-executor
		before_script:
		- python3 -m pip install twine
		- pip install twine

		.publish_python_dev:
		extends: .publish_python
		@@ -171,4 +209,4 @@ release csp-lmc-mid image:
		refs:
		- master
		variables:
		BUILD_PATH: csp-lmc-mid/docker
		BUILD_PATH: csp-lmc-mid

csp-lmc-mid/.make/.make-release-support

0 → 100644

+106 −0

Original line number	Diff line number	Diff line
		#!/bin/bash
		#
		# Copyright 2015 Xebia Nederland B.V.
		# Modifications copyright 2019 SKA Organisation
		#
		# Licensed under the Apache License, Version 2.0 (the "License");
		# you may not use this file except in compliance with the License.
		# You may obtain a copy of the License at
		#
		# http://www.apache.org/licenses/LICENSE-2.0
		#
		# Unless required by applicable law or agreed to in writing, software
		# distributed under the License is distributed on an "AS IS" BASIS,
		# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		# See the License for the specific language governing permissions and
		# limitations under the License.
		#
		function hasChanges() {
		test -n "$(git status -s .)"
		}

		function getRelease() {
		awk -F= '/^release=/{print $2}' .release
		}

		function getBaseTag() {
		sed -n -e "s/^tag=$.*$$(getRelease)\$/\1/p" .release
		}

		function getTag() {
		if [ -z "$1" ] ; then
		awk -F= '/^tag/{print $2}' .release
		else
		echo "$(getBaseTag)$1"
		fi
		}

		function setRelease() {
		if [ -n "$1" ] ; then
		sed -i.x -e "s/^tag=.*/tag=$(getTag $1)/" .release
		sed -i.x -e "s/^release=.*/release=$1/g" .release
		rm -f .release.x
		runPreTagCommand "$1"
		else
		echo "ERROR: missing release version parameter " >&2
		return 1
		fi
		}

		function runPreTagCommand() {
		if [ -n "$1" ] ; then
		COMMAND=$(sed -n -e "s/@@RELEASE@@/$1/g" -e 's/^pre_tag_command=$.*$/\1/p' .release)
		if [ -n "$COMMAND" ] ; then
		if ! OUTPUT=$(bash -c "$COMMAND" 2>&1) ; then echo $OUTPUT >&2 && exit 1 ; fi
		fi
		else
		echo "ERROR: missing release version parameter " >&2
		return 1
		fi
		}

		function tagExists() {
		tag=${1:-$(getTag)}
		test -n "$tag" && test -n "$(git tag \| grep "^$tag\$")"
		}

		function differsFromRelease() {
		tag=$(getTag)
		! tagExists $tag \|\| test -n "$(git diff --shortstat -r $tag .)"
		}

		function getVersion() {
		result=$(getRelease)

		if differsFromRelease; then
		result="$result-$(git log -n 1 --format=%h .)"
		fi

		if hasChanges ; then
		result="$result-dirty"
		fi
		echo $result
		}

		function nextPatchLevel() {
		version=${1:-$(getRelease)}
		major_and_minor=$(echo $version \| cut -d. -f1,2)
		patch=$(echo $version \| cut -d. -f3)
		version=$(printf "%s.%d" $major_and_minor $(($patch + 1)))
		echo $version
		}

		function nextMinorLevel() {
		version=${1:-$(getRelease)}
		major=$(echo $version \| cut -d. -f1);
		minor=$(echo $version \| cut -d. -f2);
		version=$(printf "%d.%d.0" $major $(($minor + 1))) ;
		echo $version
		}

		function nextMajorLevel() {
		version=${1:-$(getRelease)}
		major=$(echo $version \| cut -d. -f1);
		version=$(printf "%d.0.0" $(($major + 1)))
		echo $version
		}

csp-lmc-mid/.make/k8s.mk

0 → 100644

+283 −0

Original line number	Diff line number	Diff line
		IMAGE_TO_RUN_TEST ?= nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
		k8s: ## Which kubernetes are we connected to
		@echo "Kubernetes cluster-info:"
		@kubectl cluster-info
		@echo ""
		@echo "kubectl version:"
		@kubectl version
		@echo ""
		@echo "Helm version:"
		@helm version --client

		namespace: ## create the kubernetes namespace
		kubectl describe namespace $(KUBE_NAMESPACE) \|\| kubectl create namespace $(KUBE_NAMESPACE)

		delete_namespace: ## delete the kubernetes namespace
		@if [ "default" == "$(KUBE_NAMESPACE)" ] \|\| [ "kube-system" == "$(KUBE_NAMESPACE)" ]; then \
		echo "You cannot delete Namespace: $(KUBE_NAMESPACE)"; \
		exit 1; \
		else \
		kubectl describe namespace $(KUBE_NAMESPACE) && kubectl delete namespace $(KUBE_NAMESPACE); \
		fi

		deploy: namespace mkcerts depends ## deploy the helm chart
		@helm install $(HELM_RELEASE) charts/$(HELM_CHART)/ \
		--namespace $(KUBE_NAMESPACE) \
		--set xauthority="$(XAUTHORITYx)" \
		--set display="$(DISPLAY)" \
		--set ingress.hostname=$(INGRESS_HOST) \
		--set helmTests=false

		delete: ## delete the helm chart release
		@helm uninstall $(HELM_RELEASE) --namespace $(KUBE_NAMESPACE)

		install: namespace mkcerts ## install the helm chart
		@helm install $(HELM_RELEASE) charts/$(HELM_CHART)/ \
		--namespace $(KUBE_NAMESPACE) \
		--set xauthority="$(XAUTHORITYx)" \
		--set display="$(DISPLAY)" \
		--set ingress.hostname=$(INGRESS_HOST)

		show: mkcerts ## show the helm chart
		@helm template $(HELM_RELEASE) charts/$(HELM_CHART)/ \
		--namespace $(KUBE_NAMESPACE) \
		--set xauthority="$(XAUTHORITYx)" \
		--set display="$(DISPLAY)" \
		--set ingress.hostname=$(INGRESS_HOST)

		chart_lint: ## lint check the helm chart
		@helm lint charts/$(HELM_CHART)/ \
		--namespace $(KUBE_NAMESPACE) \
		--set ingress.hostname=$(INGRESS_HOST) \
		--set xauthority="$(XAUTHORITYx)" \
		--set display="$(DISPLAY)" \

		helm_delete: ## delete the helm chart release (with Tiller)
		@helm delete $(HELM_RELEASE) --purge \


		traefik: ## install the helm chart for traefik (in the kube-system namespace). @param: EXTERNAL_IP (i.e. private ip of the master node).
		@TMP=`mktemp -d`; \
		$(helm_add_stable_repo) && \
		helm fetch stable/traefik --untar --untardir $$TMP && \
		helm template $(helm_install_shim) $$TMP/traefik -n traefik0 --namespace kube-system \
		--set externalIP="$(EXTERNAL_IP)" \
		\| kubectl apply -n kube-system -f - && \
		rm -rf $$TMP ; \


		delete_traefik: ## delete the helm chart for traefik. @param: EXTERNAL_IP
		@TMP=`mktemp -d`; \
		$(helm_add_stable_repo) && \
		helm fetch stable/traefik --untar --untardir $$TMP && \
		helm template $(helm_install_shim) $$TMP/traefik -n traefik0 --namespace kube-system \
		--set externalIP="$(EXTERNAL_IP)" \
		\| kubectl delete -n kube-system -f - && \
		rm -rf $$TMP

		describe: ## describe Pods executed from Helm chart
		@for i in `kubectl -n $(KUBE_NAMESPACE) get pods -l app.kubernetes.io/instance=$(HELM_RELEASE) -o=name`; \
		do echo "---------------------------------------------------"; \
		echo "Describe for $${i}"; \
		echo kubectl -n $(KUBE_NAMESPACE) describe $${i}; \
		echo "---------------------------------------------------"; \
		kubectl -n $(KUBE_NAMESPACE) describe $${i}; \
		echo "---------------------------------------------------"; \
		echo ""; echo ""; echo ""; \
		done

		logs: ## show Helm chart POD logs
		@for i in `kubectl -n $(KUBE_NAMESPACE) get pods -l app.kubernetes.io/instance=$(HELM_RELEASE) -o=name`; \
		do \
		echo "---------------------------------------------------"; \
		echo "Logs for $${i}"; \
		echo kubectl -n $(KUBE_NAMESPACE) logs $${i}; \
		echo kubectl -n $(KUBE_NAMESPACE) get $${i} -o jsonpath="{.spec.initContainers[*].name}"; \
		echo "---------------------------------------------------"; \
		for j in `kubectl -n $(KUBE_NAMESPACE) get $${i} -o jsonpath="{.spec.initContainers[*].name}"`; do \
		RES=`kubectl -n $(KUBE_NAMESPACE) logs $${i} -c $${j} 2>/dev/null`; \
		echo "initContainer: $${j}"; echo "$${RES}"; \
		echo "---------------------------------------------------";\
		done; \
		echo "Main Pod logs for $${i}"; \
		echo "---------------------------------------------------"; \
		for j in `kubectl -n $(KUBE_NAMESPACE) get $${i} -o jsonpath="{.spec.containers[*].name}"`; do \
		RES=`kubectl -n $(KUBE_NAMESPACE) logs $${i} -c $${j} 2>/dev/null`; \
		echo "Container: $${j}"; echo "$${RES}"; \
		echo "---------------------------------------------------";\
		done; \
		echo "---------------------------------------------------"; \
		echo ""; echo ""; echo ""; \
		done

		localip: ## set local Minikube IP in /etc/hosts file for Ingress $(INGRESS_HOST)
		@new_ip=`minikube ip` && \
		existing_ip=`grep $(INGRESS_HOST) /etc/hosts \|\| true` && \
		echo "New IP is: $${new_ip}" && \
		echo "Existing IP: $${existing_ip}" && \
		if [ -z "$${existing_ip}" ]; then echo "$${new_ip} $(INGRESS_HOST)" \| sudo tee -a /etc/hosts; \
		else sudo perl -i -ne "s/\d+\.\d+.\d+\.\d+/$${new_ip}/ if /$(INGRESS_HOST)/; print" /etc/hosts; fi && \
		echo "/etc/hosts is now: " `grep $(INGRESS_HOST) /etc/hosts`

		mkcerts: ## Make dummy certificates for $(INGRESS_HOST) and Ingress
		@if [ ! -f charts/$(HELM_CHART)/secrets/tls.key ]; then \
		openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 \
		-keyout charts/$(HELM_CHART)/secrets/tls.key \
		-out charts/$(HELM_CHART)/secrets/tls.crt \
		-subj "/CN=$(INGRESS_HOST)/O=Minikube"; \
		else \
		echo "SSL cert already exits in charts/$(HELM_CHART)/secrets ... skipping"; \
		fi

		# Utility target to install Helm dependencies
		helm_dependencies:
		@which helm ; rc=$$?; \
		if [[ $$rc != 0 ]]; then \
		curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3; \
		chmod 700 get_helm.sh; \
		./get_helm.sh; \
		fi; \
		helm version --client

		# Utility target to install K8s dependencies
		kubectl_dependencies:
		@([ -n "$(KUBE_CONFIG_BASE64)" ] && [ -n "$(KUBECONFIG)" ]) \|\| (echo "unset variables [KUBE_CONFIG_BASE64/KUBECONFIG] - abort!"; exit 1)
		@which kubectl ; rc=$$?; \
		if [[ $$rc != 0 ]]; then \
		curl -L -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/$(KUBERNETES_VERSION)/bin/linux/amd64/kubectl"; \
		chmod +x /usr/bin/kubectl; \
		mkdir -p /etc/deploy; \
		echo $(KUBE_CONFIG_BASE64) \| base64 -d > $(KUBECONFIG); \
		fi
		@echo -e "\nkubectl client version:"
		@kubectl version --client
		@echo -e "\nkubectl config view:"
		@kubectl config view
		@echo -e "\nkubectl config get-contexts:"
		@kubectl config get-contexts
		@echo -e "\nkubectl version:"
		@kubectl version

		kubeconfig: ## export current KUBECONFIG as base64 ready for KUBE_CONFIG_BASE64
		@KUBE_CONFIG_BASE64=`kubectl config view --flatten \| base64 -w 0`; \
		echo "KUBE_CONFIG_BASE64: $$(echo $${KUBE_CONFIG_BASE64} \| cut -c 1-40)..."; \
		echo "appended to: PrivateRules.mak"; \
		echo -e "\n\n# base64 encoded from: kubectl config view --flatten\nKUBE_CONFIG_BASE64 = $${KUBE_CONFIG_BASE64}" >> PrivateRules.mak

		wait:
		@echo "Waiting for device servers to be ready"
		@date
		@kubectl -n $(KUBE_NAMESPACE) get pods -l cspServer
		@kubectl -n $(KUBE_NAMESPACE) wait --for=condition=ready --timeout=120s -l cspServer pods
		@date

		#
		# defines a function to copy the ./test-harness directory into the K8s TEST_RUNNER
		# and then runs the requested make target in the container.
		# capture the output of the test in a tar file
		# stream the tar file base64 encoded to the Pod logs
		#
		k8s_test = tar -c . \| \
		kubectl run $(TEST_RUNNER) \
		--namespace $(KUBE_NAMESPACE) -i --wait --restart=Never \
		--image-pull-policy=IfNotPresent \
		--image=$(IMAGE_TO_RUN_TEST) -- \
		/bin/bash -c "tar xv --strip-components 1 --warning=all && \
		python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt &&\
		python3 -m pip install . &&\
		cd test-harness &&\
		make TANGO_HOST=databaseds-tango-base-$(HELM_RELEASE):10000 $1 && \
		tar -czvf /tmp/build.tgz build && \
		echo '~~~~BOUNDARY~~~~' && \
		cat /tmp/build.tgz \| base64 && \
		echo '~~~~BOUNDARY~~~~'" \
		2>&1

		# run the test function
		# save the status
		# clean out build dir
		# print the logs minus the base64 encoded payload
		# pull out the base64 payload and unpack build/ dir
		# base64 payload is given a boundary "~~~~BOUNDARY~~~~" and extracted using perl
		# clean up the run to completion container
		# exit the saved status
		k8s_test: deploy wait## test the application on K8s
		@echo "KUBE_NAMESPACE: $(KUBE_NAMESPACE)"
		$(call k8s_test,test); \
		status=$$?; \
		rm -fr build; \
		kubectl --namespace $(KUBE_NAMESPACE) logs $(TEST_RUNNER) \| perl -ne 'BEGIN {$$on=1;}; if (index($$_, "~~~~BOUNDARY~~~~")!=-1){$$on+=1;next;}; print if $$on % 2;'; \
		kubectl --namespace $(KUBE_NAMESPACE) logs $(TEST_RUNNER) \| \
		perl -ne 'BEGIN {$$on=0;}; if (index($$_, "~~~~BOUNDARY~~~~")!=-1){$$on+=1;next;}; print if $$on % 2;' \| \
		base64 -d \| tar -xzf -; \
		kubectl --namespace $(KUBE_NAMESPACE) delete pod $(TEST_RUNNER); \
		exit $$status


		rlint: ## run lint check on Helm Chart using gitlab-runner
		if [ -n "$(RDEBUG)" ]; then DEBUG_LEVEL=debug; else DEBUG_LEVEL=warn; fi && \
		gitlab-runner --log-level $${DEBUG_LEVEL} exec $(EXECUTOR) \
		--docker-privileged \
		--docker-disable-cache=false \
		--docker-host $(DOCKER_HOST) \
		--docker-volumes $(DOCKER_VOLUMES) \
		--docker-pull-policy always \
		--timeout $(TIMEOUT) \
		--env "DOCKER_HOST=$(DOCKER_HOST)" \
		--env "DOCKER_REGISTRY_USER_LOGIN=$(DOCKER_REGISTRY_USER_LOGIN)" \
		--env "CI_REGISTRY_PASS_LOGIN=$(CI_REGISTRY_PASS_LOGIN)" \
		--env "CI_REGISTRY=$(CI_REGISTRY)" \
		lint-check-chart \|\| true

		# K8s testing with local gitlab-runner
		# Run the powersupply tests in the TEST_RUNNER run to completion Pod:
		# set namespace
		# install dependencies for Helm and kubectl
		# deploy into namespace
		# run test in run to completion Pod
		# extract Pod logs
		# set test return code
		# delete
		# delete namespace
		# return result
		rk8s_test: ## run k8s_test on K8s using gitlab-runner
		if [ -n "$(RDEBUG)" ]; then DEBUG_LEVEL=debug; else DEBUG_LEVEL=warn; fi && \
		KUBE_NAMESPACE=`git rev-parse --abbrev-ref HEAD \| tr -dc 'A-Za-z0-9\-' \| tr '[:upper:]' '[:lower:]'` && \
		gitlab-runner --log-level $${DEBUG_LEVEL} exec $(EXECUTOR) \
		--docker-privileged \
		--docker-disable-cache=false \
		--docker-host $(DOCKER_HOST) \
		--docker-volumes $(DOCKER_VOLUMES) \
		--docker-pull-policy always \
		--timeout $(TIMEOUT) \
		--env "DOCKER_HOST=$(DOCKER_HOST)" \
		--env "DOCKER_REGISTRY_USER_LOGIN=$(DOCKER_REGISTRY_USER_LOGIN)" \
		--env "CI_REGISTRY_PASS_LOGIN=$(CI_REGISTRY_PASS_LOGIN)" \
		--env "CI_REGISTRY=$(CI_REGISTRY)" \
		--env "KUBE_CONFIG_BASE64=$(KUBE_CONFIG_BASE64)" \
		--env "KUBECONFIG=$(KUBECONFIG)" \
		--env "KUBE_NAMESPACE=$${KUBE_NAMESPACE}" \
		test-chart \|\| true


		helm_tests: ## run Helm chart tests
		helm test $(HELM_RELEASE) --cleanup

		ingress_check: ## curl test Tango REST API - https://tango-controls.readthedocs.io/en/latest/development/advanced/rest-api.html#tango-rest-api-implementations
		@echo "---------------------------------------------------"
		@echo "Test HTTP:"; echo ""
		curl -u "tango-cs:tango" -XGET http://$(INGRESS_HOST)/tango/rest/rc4/hosts/databaseds-tango-example-$(HELM_RELEASE)/10000 \| json_pp
		@echo "", echo ""
		@echo "---------------------------------------------------"
		@echo "Test HTTPS:"; echo ""
		curl -k -u "tango-cs:tango" -XGET https://$(INGRESS_HOST)/tango/rest/rc4/hosts/databaseds-tango-example-$(HELM_RELEASE)/10000 \| json_pp
		@echo ""

		help: ## show this help.
		@echo "make targets:"
		@grep -hE '^[0-9a-zA-Z_-]+:.?## .$$' $(MAKEFILE_LIST) \| sort \| awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
		@echo ""; echo "make vars (+defaults):"
		@grep -hE '^[0-9a-zA-Z_-]+ \?=.*$$' $(MAKEFILE_LIST) \| sort \| awk 'BEGIN {FS = " \?\= "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' \| sed -e 's/\#\#/ \#/'

		depends:
		helm dependency update charts/$(HELM_CHART)/

csp-lmc-mid/.make/postdeploy.mk

0 → 100644

+130 −0

Original line number	Diff line number	Diff line
		.PHONY: k8s_test smoketest template_tests tango_rest_ingress_check

		#
		# IMAGE_TO_TEST defines the tag of the Docker image to test
		#
		IMAGE_TO_TEST ?= nexus.engageska-portugal.pt/ska-docker/tango-vscode:0.2.4## docker image that will be run for testing purpose
		#IMAGE_TO_TEST = $(DOCKER_REGISTRY_HOST)/$(DOCKER_REGISTRY_USER)/$(PROJECT):latest
		# Test runner - run to completion job in K8s
		TEST_RUNNER = test-makefile-runner-$(CI_JOB_ID)-$(KUBE_NAMESPACE)-$(HELM_RELEASE)##name of the pod running the k8s_tests
		#
		# defines a function to copy the ./test-harness directory into the K8s TEST_RUNNER
		# and then runs the requested make target in the container.
		# capture the output of the test in a build folder inside the container
		#
		TANGO_HOST = databaseds-tango-base-$(HELM_RELEASE):10000
		MARK ?= fast## this will allow to add the mark parameter of pytest
		SLEEPTIME ?= 30s ##amount of sleep time for the smoketest target

		#
		# defines a function to copy the ./test-harness directory into the K8s TEST_RUNNER
		# and then runs the requested make target in the container.
		# capture the output of the test in a tar file
		# stream the tar file base64 encoded to the Pod logs
		#
		ktest = tar -c post-deployment/ \| \
		kubectl run $(TEST_RUNNER) \
		--namespace $(KUBE_NAMESPACE) -i --wait --restart=Never \
		--image-pull-policy=IfNotPresent \
		--image=$(IMAGE_TO_TEST) -- \
		/bin/bash -c "mkdir csp-lmc-mid && tar xv --directory csp-lmc-mid --strip-components 1 --warning=all && cd csp-lmc-mid && \
		make KUBE_NAMESPACE=$(KUBE_NAMESPACE) HELM_RELEASE=$(HELM_RELEASE) TANGO_HOST=$(TANGO_HOST) MARK=$(MARK) $1 && \
		tar -czvf /tmp/build.tgz build && \
		echo '~~~~BOUNDARY~~~~' && \
		cat /tmp/build.tgz \| base64 && \
		echo '~~~~BOUNDARY~~~~'" \
		2>&1

		# run the test function
		# save the status
		# clean out build dir
		# print the logs minus the base64 encoded payload
		# pull out the base64 payload and unpack build/ dir
		# base64 payload is given a boundary "~~~~BOUNDARY~~~~" and extracted using perl
		# clean up the run to completion container
		# exit the saved status
		ktest: smoketest## test the application on K8s
		$(call ktest,test); \
		status=$$?; \
		rm -fr build; \
		kubectl --namespace $(KUBE_NAMESPACE) logs $(TEST_RUNNER) \| \
		perl -ne 'BEGIN {$$on=0;}; if (index($$_, "~~~~BOUNDARY~~~~")!=-1){$$on+=1;next;}; print if $$on % 2;' \| \
		base64 -d \| tar -xzf -; \
		kubectl --namespace $(KUBE_NAMESPACE) delete pod $(TEST_RUNNER); \
		exit $$status


		smoketest: ## check that the number of waiting containers is zero (10 attempts, wait time 30s).
		@echo "Smoke test START"; \
		n=10; \
		while [ $$n -gt 0 ]; do \
		waiting=`kubectl get pods -n $(KUBE_NAMESPACE) -o=jsonpath='{.items[].status.containerStatuses[].state.waiting.reason}' \| wc -w`; \
		echo "Waiting containers=$$waiting"; \
		if [ $$waiting -ne 0 ]; then \
		echo "Waiting $(SLEEPTIME) for pods to become running...#$$n"; \
		sleep $(SLEEPTIME); \
		fi; \
		if [ $$waiting -eq 0 ]; then \
		echo "Smoke test SUCCESS"; \
		exit 0; \
		fi; \
		if [ $$n -eq 1 ]; then \
		waiting=`kubectl get pods -n $(KUBE_NAMESPACE) -o=jsonpath='{.items[].status.containerStatuses[].state.waiting.reason}' \| wc -w`; \
		echo "Smoke test FAILS"; \
		echo "Found $$waiting waiting containers: "; \
		kubectl get pods -n $(KUBE_NAMESPACE) -o=jsonpath='{range .items[*].status.containerStatuses[?(.state.waiting)]}{.state.waiting.message}{"\n"}{end}'; \
		exit 1; \
		fi; \
		n=`expr $$n - 1`; \
		done

		template_tests:
		rc=0; \
		for chrt in `ls charts/`; do \
		helm unittest -f template_tests/*_test.yaml charts/$$chrt \
		\|\| rc=2 && continue; \
		done; \
		exit $$rc

		tango_rest_ingress_check: ## curl test Tango REST API - https://tango-controls.readthedocs.io/en/latest/development/advanced/rest-api.html#tango-rest-api-implementations
		@echo "---------------------------------------------------"
		@echo "Test HTTP:"; echo ""
		curl -u "tango-cs:tango" -XGET http://tango.rest.$(INGRESS_HOST)/tango/rest/rc4/hosts/databaseds-tango-base-$(HELM_RELEASE)/10000 \| json_pp
		# @echo "", echo ""
		# @echo "---------------------------------------------------"
		# @echo "Test HTTPS:"; echo ""
		# curl -k -u "tango-cs:tango" -XGET https://tango.rest.$(INGRESS_HOST)/tango/rest/rc4/hosts/databaseds-tango-base-$(HELM_RELEASE)/10000 \| json_pp
		# @echo ""

		oet_podname = $(shell kubectl get pods -l app=rest-oet-$(HELM_RELEASE) -o=jsonpath='{..metadata.name}')
		sut_cdm_ver= $(shell kubectl exec -it $(oet_podname) pip list \| grep "cdm-shared-library" \| awk ' {print $$2}' \| awk 'BEGIN { FS = "+" } ; {print $$1}')
		sut_cdm_cur_ver=$(shell grep "cdm-shared-library" post-deployment/SUT_requirements.txt \| awk 'BEGIN { FS = "==" } ; {print $$2}')
		sut_oet_ver = $(shell kubectl exec -it $(oet_podname) pip list \| grep "observation-execution-tool" \| awk ' {print $$2}' \| awk 'BEGIN { FS = "+" } ; {print $$1}')
		sut_oet_cur_ver=$(shell grep "observation-execution-tool" post-deployment/SUT_requirements.txt \| awk 'BEGIN { FS = "==" } ; {print $$2}')

		check_oet_packages:
		@echo "MVP is based on cdm-shared-library=$(sut_cdm_ver)"
		@echo "Test are based on cdm-shared-library=$(sut_cdm_cur_ver)"
		@if [ $(sut_cdm_ver) != $(sut_cdm_cur_ver) ] ; then \
		echo "Warning: cdm-shared-library package for MVP is not the same as used for testing!"; \
		fi
		@echo "MVP is based on observation-execution-tool=$(sut_oet_ver)"
		@echo "Test are based on observation-execution-tool=$(sut_oet_cur_ver)"
		@if [ $(sut_oet_ver) != $(sut_oet_cur_ver) ] ; then \
		echo "Warning: observation-execution-tool package for MVP is not the same as used for testing!"; \
		fi

		##the following section is for developers requiring the testing pod to be instantiated with a volume mappig to skampi
		-include dev-testing.mk

		timestamp=$(shell date -u +"%s")

		sleepy_time:
		sleep 5s

		measure_time:
		time1=$$SECONDS; \
		make sleepy_time; \
		time2=$$SECONDS; \
		elapsed=$$((time2 - time1)); \
		echo $$elapsed