Added the option to disable the SSL certificate check in the agent. (36f4f515) · Commits · ExaCT / Rosetta

docker-compose-dev.yml

+1 −0

Original line number	Original line	Diff line number	Diff line
	@@ -69,6 +69,7 @@ services:
	- DJANGO_DEBUG=True		- DJANGO_DEBUG=True
	- DJANGO_LOG_LEVEL=CRITICAL		- DJANGO_LOG_LEVEL=CRITICAL
	- ROSETTA_LOG_LEVEL=DEBUG		- ROSETTA_LOG_LEVEL=DEBUG
			#- ROSETTA_AGENT_CHECK_SSL=True # If the agent should check the SSL certificate
	#- ROSETTA_WEBAPP_SSL=False # Used for the agent.		#- ROSETTA_WEBAPP_SSL=False # Used for the agent.
	#- ROSETTA_WEBAPP_HOST= # Used for the agent, can be internal. Defaults to webapp IP.		#- ROSETTA_WEBAPP_HOST= # Used for the agent, can be internal. Defaults to webapp IP.
	#- ROSETTA_WEBAPP_PORT=8080 # Used for the agent, can be internal.		#- ROSETTA_WEBAPP_PORT=8080 # Used for the agent, can be internal.

services/webapp/code/rosetta/core_app/api.py

+15 −3

Original line number	Original line	Diff line number	Diff line
	@@ -11,7 +11,7 @@ from django.conf import settings
	from rest_framework.response import Response		from rest_framework.response import Response
	from rest_framework import status, serializers, viewsets		from rest_framework import status, serializers, viewsets
	from rest_framework.views import APIView		from rest_framework.views import APIView
	from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access_mode_credentials, get_or_create_container_from_repository		from .utils import format_exception, send_email, os_shell, now_t, get_ssh_access_mode_credentials, get_or_create_container_from_repository, booleanize
	from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair		from .models import Profile, Task, TaskStatuses, Computing, Storage, KeyPair
	from .exceptions import ConsistencyException		from .exceptions import ConsistencyException
	import json		import json
	@@ -19,6 +19,7 @@ import json
	# Setup logging		# Setup logging
	logger = logging.getLogger(__name__)		logger = logging.getLogger(__name__)

			ROSETTA_AGENT_CHECK_SSL = booleanize(os.environ.get('ROSETTA_AGENT_CHECK_SSL', True))

	#==============================		#==============================
	# Common returns		# Common returns
	@@ -252,7 +253,8 @@ import socket
	try:		try:
	from urllib.request import urlopen		from urllib.request import urlopen
	except ImportError:		except ImportError:
	from urllib import urlopen		from urllib2 import urlopen
			import ssl

	# Setup logging		# Setup logging
	logger = logging.getLogger('Agent')		logger = logging.getLogger('Agent')
	@@ -260,6 +262,8 @@ logging.basicConfig(level=logging.INFO)

	hostname = socket.gethostname()		hostname = socket.gethostname()

			check_ssl='''+str(ROSETTA_AGENT_CHECK_SSL)+'''

	# Task id set by the API		# Task id set by the API
	task_uuid = "'''+ task_uuid +'''"		task_uuid = "'''+ task_uuid +'''"

	@@ -292,8 +296,16 @@ while True:
	break		break
	logger.info(' - ports: "{},{},{}"'.format(port, port+1, port+2))		logger.info(' - ports: "{},{},{}"'.format(port, port+1, port+2))

			if not check_ssl:
			context = ssl.create_default_context()
			context.check_hostname = False
			context.verify_mode = ssl.CERT_NONE
			response = urlopen("'''+webapp_conn_string+'''/api/v1/base/agent/?task_uuid={}&action=set_ip_port&ip={}&port={}".format(task_uuid, ip, port), context=context)
			else:
	response = urlopen("'''+webapp_conn_string+'''/api/v1/base/agent/?task_uuid={}&action=set_ip_port&ip={}&port={}".format(task_uuid, ip, port))		response = urlopen("'''+webapp_conn_string+'''/api/v1/base/agent/?task_uuid={}&action=set_ip_port&ip={}&port={}".format(task_uuid, ip, port))

	response_content = response.read()		response_content = response.read()

	if response_content not in ['OK', b'OK']:		if response_content not in ['OK', b'OK']:
	logger.error(response_content)		logger.error(response_content)
	logger.info('Not everything OK, exiting with status code =1')		logger.info('Not everything OK, exiting with status code =1')

services/webapp/code/rosetta/core_app/computing_managers.py

+7 −4

Original line number	Original line	Diff line number	Diff line
			import os
	from .models import TaskStatuses, KeyPair, Task, Storage		from .models import TaskStatuses, KeyPair, Task, Storage
	from .utils import os_shell, get_ssh_access_mode_credentials, sanitize_container_env_vars		from .utils import os_shell, get_ssh_access_mode_credentials, sanitize_container_env_vars, booleanize
	from .exceptions import ErrorMessage, ConsistencyException		from .exceptions import ErrorMessage, ConsistencyException
	from django.conf import settings		from django.conf import settings

	@@ -7,6 +8,8 @@ from django.conf import settings
	import logging		import logging
	logger = logging.getLogger(__name__)		logger = logging.getLogger(__name__)

			ROSETTA_AGENT_CHECK_SSL = booleanize(os.environ.get('ROSETTA_AGENT_CHECK_SSL', True))
			CHECK_WGET_CERT_STR = '--no-check-certificate' if not ROSETTA_AGENT_CHECK_SSL else ''

	class ComputingManager(object):		class ComputingManager(object):

	@@ -269,7 +272,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana

	run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)		run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)
	run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid)		run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid)
	run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)		run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
	run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT {} {} &&'.format(authstring, varsstring)		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT {} {} &&'.format(authstring, varsstring)
	run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)

	@@ -332,7 +335,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana

	run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)		run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)
	run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid)		run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid)
	run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)		run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
	run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds)		run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds)
	if container_engine == 'podman':		if container_engine == 'podman':
	run_command += '--network=private --uts=private --userns=keep-id '		run_command += '--network=private --uts=private --userns=keep-id '
	@@ -522,7 +525,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
	binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)		binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)

	run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)		run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)
	run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)		run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {} {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
	run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT {} {} && '.format(authstring, varsstring)		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT {} {} && '.format(authstring, varsstring)
	run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)		run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
	run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)