Refactored models, parts of the UI and internal logics. (49926bf8) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/api.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -322,7 +322,7 @@ print(port)
		logger.info('Setting task "{}" to ip "{}" and port "{}"'.format(task.uuid, task_ip, task_port))
		task.status = TaskStatuses.running
		task.ip = task_ip
		if task.container.supports_dynamic_ports:
		if task.container.supports_custom_interface_port:
		task.port = int(task_port)
		task.save()

services/webapp/code/rosetta/core_app/computing_managers.py

+19 −28

Original line number	Diff line number	Diff line
		@@ -41,13 +41,13 @@ class ComputingManager(object):
		task.save()

		# Check if the tunnel is active and if so kill it
		logger.debug('Checking if task "{}" has a running tunnel'.format(task.tid))
		check_command = 'ps -ef \| grep ":'+str(task.tunnel_port)+':'+str(task.ip)+':'+str(task.port)+'" \| grep -v grep \| awk \'{print $2}\''
		logger.debug('Checking if task "{}" has a running tunnel'.format(task.uuid))
		check_command = 'ps -ef \| grep ":'+str(task.tcp_tunnel_port)+':'+str(task.interface_ip)+':'+str(task.interface_port)+'" \| grep -v grep \| awk \'{print $2}\''
		logger.debug(check_command)
		out = os_shell(check_command, capture=True)
		logger.debug(out)
		if out.exit_code == 0:
		logger.debug('Task "{}" has a running tunnel, killing it'.format(task.tid))
		logger.debug('Task "{}" has a running tunnel, killing it'.format(task.uuid))
		tunnel_pid = out.stdout
		# Kill Tunnel command
		kill_tunnel_command= 'kill -9 {}'.format(tunnel_pid)
		@@ -91,15 +91,12 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):

		def _start_task(self, task):

		if task.container.type != 'docker':
		raise ErrorMessage('Sorry, only Docker container are supported on this computing resource.')

		# Init run command #--cap-add=NET_ADMIN --cap-add=NET_RAW
		run_command = 'sudo docker run --network=rosetta_default --name rosetta-task-{}'.format( task.id)
		run_command = 'sudo docker run --network=rosetta_default --name {}'.format(task.uuid)

		# Pass if any
		if task.auth_pass:
		run_command += ' -eAUTH_PASS={} '.format(task.auth_pass)
		if task.password:
		run_command += ' -eAUTH_PASS={} '.format(task.password)

		# User data volume
		run_command += ' -v {}/user-{}:/data'.format(settings.LOCAL_USER_DATA_DIR, task.user.id)
		@@ -108,10 +105,10 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
		if task.container.registry == 'local':
		registry_string = 'localhost:5000/'
		else:
		registry_string = ''
		registry_string = 'docker.io/'

		# Host name, image entry command
		run_command += ' -h task-{} -d -t {}{}'.format(task.id, registry_string, task.container.image)
		run_command += ' -h task-{} -d -t {}{}'.format(task.uuid, registry_string, task.container.image)

		# Debug
		logger.debug('Running new task with command="{}"'.format(run_command))
		@@ -121,20 +118,20 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
		if out.exit_code != 0:
		raise Exception(out.stderr)
		else:
		task_tid = out.stdout
		logger.debug('Created task with id: "{}"'.format(task_tid))
		tid = out.stdout
		logger.debug('Created task with id: "{}"'.format(tid))

		# Get task IP address
		out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + task_tid + ' \| tail -n1', capture=True)
		out = os_shell('sudo docker inspect --format \'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' ' + tid + ' \| tail -n1', capture=True)
		if out.exit_code != 0:
		raise Exception('Error: ' + out.stderr)
		task_ip = out.stdout

		# Set fields
		task.tid = task_tid
		task.id = tid
		task.status = TaskStatuses.running
		task.ip = task_ip
		task.port = int(task.container.ports.split(',')[0])
		task.interface_ip = task_ip
		task.interface_port = task.container.interface_port

		# Save
		task.save()
		@@ -145,9 +142,9 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
		# Delete the Docker container
		standby_supported = False
		if standby_supported:
		stop_command = 'sudo docker stop {}'.format(task.tid)
		stop_command = 'sudo docker stop {}'.format(task.id)
		else:
		stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.tid,task.tid)
		stop_command = 'sudo docker stop {} && sudo docker rm {}'.format(task.id,task.id)

		out = os_shell(stop_command, capture=True)
		if out.exit_code != 0:
		@@ -164,7 +161,7 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
		def _get_task_log(self, task, **kwargs):

		# View the Docker container log (attach)
		view_log_command = 'sudo docker logs {}'.format(task.tid,)
		view_log_command = 'sudo docker logs {}'.format(task.id,)
		logger.debug(view_log_command)
		out = os_shell(view_log_command, capture=True)
		if out.exit_code != 0:
		@@ -200,8 +197,6 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
		# Run the container on the host (non blocking)
		if task.container.type == 'singularity':

		task.tid = task.uuid
		task.save()

		# Set pass if any
		if task.auth_pass:
		@@ -260,11 +255,7 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
		task = Task.objects.get(uuid=task_uuid)

		# Save pid echoed by the command above
		task_pid = out.stdout

		# Set fields
		#task.status = TaskStatuses.running
		task.pid = task_pid
		task.id = out.stdout

		# Save
		task.save()
		@@ -359,7 +350,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
		# Submit the job
		if task.container.type == 'singularity':

		#if not task.container.supports_dynamic_ports:
		#if not task.container.supports_custom_interface_port:
		# raise Exception('This task does not support dynamic port allocation and is therefore not supported using singularity on Slurm')

		# Set pass if any

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+72 −134

Original line number	Diff line number	Diff line
		@@ -63,116 +63,53 @@ class Command(BaseCommand):
		Text.objects.create(id='home', content=default_home_text_content)


		# Public containers
		public_containers = Container.objects.filter(user=None)
		if public_containers:
		# Platform containers
		platform_containers = Container.objects.filter(user=None)
		if platform_containers:
		print('Not creating public containers as they already exist')
		else:
		print('Creating public containers...')
		print('Creating platform containers...')

		# MinimalMetaDesktop Docker (sarusso repo)
		Container.objects.create(user = None,
		name = 'MinimalMetaDesktop latest',
		name = 'MinimalMetaDesktop ',
		description = 'A minimal meta-desktop environment providing basic window management functionalities and a terminal.',
		registry = 'docker.io',
		image = 'sarusso/minimalmetadesktop',
		type = 'docker',
		registry = 'docker_hub',
		ports = '8590',
		protocol = 'https',
		supports_dynamic_ports = True,
		supports_user_auth = False,
		supports_pass_auth = True)

		# BasicMetaDesktop Docker (sarusso repo)
		Container.objects.create(user = None,
		name = 'BasicMetaDesktop latest',
		image = 'sarusso/basicmetadesktop',
		type = 'docker',
		registry = 'docker_hub',
		protocol = 'https',
		ports = '8590',
		supports_dynamic_ports = True,
		supports_user_auth = False,
		supports_pass_auth = True)


		# DevMetaDesktop Docker (sarusso repo)
		Container.objects.create(user = None,
		name = 'DevMetaDesktop latest',
		image = 'sarusso/devmetadesktop',
		type = 'docker',
		registry = 'docker_hub',
		protocol = 'https',
		ports = '8590',
		supports_dynamic_ports = True,
		supports_user_auth = False,
		supports_pass_auth = True)


		# MinimalMetaDesktop Singularity (sarusso repo)
		Container.objects.create(user = None,
		name = 'MinimalMetaDesktop latest',
		image = 'sarusso/minimalmetadesktop',
		type = 'singularity',
		registry = 'docker_hub',
		protocol = 'https',
		ports = '8590',
		supports_dynamic_ports = True,
		supports_user_auth = False,
		supports_pass_auth = True)


		# BasicMetaDesktop Singularity (sarusso repo)
		Container.objects.create(user = None,
		name = 'BasicMetaDesktop latest',
		image = 'sarusso/basicmetadesktop',
		type = 'singularity',
		registry = 'docker_hub',
		protocol = 'https',
		ports = '8590',
		supports_dynamic_ports = True,
		supports_user_auth = False,
		supports_pass_auth = True)


		# DevMetaDesktop Singularity (sarusso repo)
		Container.objects.create(user = None,
		name = 'DevMetaDesktop latest',
		image = 'sarusso/devmetadesktop',
		type = 'singularity',
		registry = 'docker_hub',
		protocol = 'https',
		ports = '8590',
		supports_dynamic_ports = True,
		supports_user_auth = False,
		supports_pass_auth = True)


		# MetaDesktop Singularity (local)
		tag = 'v0.2.0',
		arch = 'x86_64',
		os = 'linux',
		interface_port = '8590',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = True,
		supports_interface_auth = True)

		# # BasicMetaDesktop Docker (sarusso repo)
		# Container.objects.create(user = None,
		# name = 'MetaDesktop latest',
		# image = 'rosetta/metadesktop',
		# type = 'singularity',
		# registry = 'docker_local',
		# name = 'BasicMetaDesktop latest',
		# image = 'sarusso/basicmetadesktop',
		# registry = 'docker_hub',
		# protocol = 'https',
		# ports = '8590',
		# supports_dynamic_ports = True,
		# supports_custom_interface_port = True,
		# supports_user_auth = False,
		# supports_pass_auth = True)



		# Astrocook
		#
		#
		# # DevMetaDesktop Docker (sarusso repo)
		# Container.objects.create(user = None,
		# name = 'Astrocook b2b819e',
		# image = 'sarusso/astrocook:b2b819e',
		# name = 'DevMetaDesktop latest',
		# image = 'sarusso/devmetadesktop',
		# type = 'docker',
		# registry = 'docker_local',
		# registry = 'docker_hub',
		# protocol = 'https',
		# ports = '8590',
		# supports_dynamic_ports = False,
		# supports_custom_interface_port = True,
		# supports_user_auth = False,
		# supports_pass_auth = False)

		# supports_pass_auth = True)

		# Private containers
		# Testuser containers
		testuser_containers = Container.objects.filter(user=testuser)
		if testuser_containers:
		print('Not creating testuser private containers as they already exist')
		@@ -181,25 +118,20 @@ class Command(BaseCommand):

		# Jupyter Singularity
		Container.objects.create(user = testuser,
		name = 'Jupyter Notebook latest',
		name = 'Jupyter Notebook',
		description = 'A basic Jupyter notebook environment.',
		registry = 'docker.io',
		image = 'jupyter/base-notebook',
		type = 'singularity',
		registry = 'docker_hub',
		ports = '8888',
		supports_dynamic_ports = False,
		supports_user_auth = False,
		supports_pass_auth = False)
		tag = 'latest',
		arch = 'x86_64',
		os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = False,
		supports_interface_auth = False)


		# Jupyter Docker
		Container.objects.create(user = testuser,
		name = 'Jupyter Notebook latest',
		image = 'jupyter/base-notebook',
		type = 'docker',
		registry = 'docker_hub',
		ports = '8888',
		supports_dynamic_ports = False,
		supports_user_auth = False,
		supports_pass_auth = False)

		# Computing resources
		computing_resources = Computing.objects.all()
		@@ -213,27 +145,31 @@ class Command(BaseCommand):
		#==============================
		Computing.objects.create(user = None,
		name = 'Demo Internal',
		type = 'singlenode',
		access_method = 'internal',
		description = 'A demo internal computing resource.',
		type = 'standalone',
		access_mode = 'internal',
		auth_mode = 'internal',
		wms = None,
		requires_sys_conf = False,
		requires_user_conf = False,
		requires_user_keys = False,
		supports_docker = True,
		supports_singularity = False)
		container_runtimes = 'docker')


		#==============================
		# Demo Single Node computing
		#==============================
		demo_singlenode_computing = Computing.objects.create(user = None,
		name = 'Demo Single Node',
		type = 'singlenode',
		access_method = 'ssh',
		name = 'Demo Standalone',
		description = 'A demo standalone computing resource.',
		type = 'standalone',
		access_mode = 'ssh+cli',
		auth_mode = 'user_keys',
		wms = None,
		requires_sys_conf = True,
		requires_user_conf = True,
		requires_user_keys = True,
		supports_docker = True,
		supports_singularity = True)
		container_runtimes = 'singularity')

		ComputingSysConf.objects.create(computing = demo_singlenode_computing,
		data = {'host': 'slurmclusterworker-one',
		@@ -249,13 +185,15 @@ class Command(BaseCommand):
		#==============================
		demo_slurm_computing = Computing.objects.create(user = None,
		name = 'Demo Cluster',
		description = 'A demo cluster computing resource.',
		type = 'cluster',
		access_method = 'slurm+ssh',
		access_mode = 'ssh+cli',
		auth_mode = 'user_keys',
		wms = 'slurm',
		requires_sys_conf = True,
		requires_user_conf = True,
		requires_user_keys = True,
		supports_docker = False,
		supports_singularity = True)
		container_runtimes = 'singularity')

		# Create demo slurm sys computing conf
		ComputingSysConf.objects.create(computing = demo_slurm_computing,

services/webapp/code/rosetta/core_app/migrations/0005_auto_20211030_2318.py

0 → 100644

+165 −0

File added.

Preview size limit exceeded, changes collapsed.

services/webapp/code/rosetta/core_app/migrations/0006_auto_20211102_1334.py

0 → 100644

+58 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-11-02 13:34

		from django.db import migrations, models


		class Migration(migrations.Migration):

		dependencies = [
		('core_app', '0005_auto_20211030_2318'),
		]

		operations = [
		migrations.RenameField(
		model_name='container',
		old_name='supports_password_auth',
		new_name='supports_interface_auth',
		),
		migrations.RenameField(
		model_name='container',
		old_name='supports_dynamic_ports',
		new_name='supports_setting_interface_port',
		),
		migrations.RenameField(
		model_name='task',
		old_name='tunnel_port',
		new_name='tcp_tunnel_port',
		),
		migrations.RemoveField(
		model_name='task',
		name='proxy_token',
		),
		migrations.RemoveField(
		model_name='task',
		name='requires_tunnel',
		),
		migrations.AddField(
		model_name='task',
		name='auth_token',
		field=models.CharField(blank=True, max_length=36, null=True, verbose_name='A one-time token for proxy or task interface auth if any'),
		),
		migrations.AddField(
		model_name='task',
		name='requires_proxy_auth',
		field=models.BooleanField(default=False, verbose_name='Does the task require interface authentication to be enforced at proxy-level?'),
		preserve_default=False,
		),
		migrations.AddField(
		model_name='task',
		name='requires_tcp_tunnel',
		field=models.BooleanField(default=True, verbose_name='Does the task require a tunnel to be opened for accessing its interface?'),
		preserve_default=False,
		),
		migrations.AlterField(
		model_name='task',
		name='requires_proxy',
		field=models.BooleanField(verbose_name='Does the task require a proxy for accessing its interface?'),
		),
		]