UI improvements, few changes in the models as well for the sake of clarity. (4f91f56f) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/admin.py

+2 −2

Original line number	Diff line number	Diff line
		from django.contrib import admin

		from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf, Keys
		from .models import Profile, LoginToken, Task, Container, Computing, ComputingSysConf, ComputingUserConf, KeyPair

		admin.site.register(Profile)
		admin.site.register(LoginToken)
		@@ -9,4 +9,4 @@ admin.site.register(Container)
		admin.site.register(Computing)
		admin.site.register(ComputingSysConf)
		admin.site.register(ComputingUserConf)
		admin.site.register(Keys)
		admin.site.register(KeyPair)

services/webapp/code/rosetta/core_app/computing_managers.py

+20 −30

Original line number	Diff line number	Diff line
		from .models import TaskStatuses, Keys, Task
		from .models import TaskStatuses, KeyPair, Task
		from .utils import os_shell
		from .exceptions import ErrorMessage, ConsistencyException

		@@ -164,8 +164,8 @@ class RemoteComputingManager(ComputingManager):
		user = task.computing.get_conf_param('user')

		# Get user keys
		if task.computing.require_user_keys:
		user_keys = Keys.objects.get(user=task.user, default=True)
		if task.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		@@ -173,9 +173,7 @@ class RemoteComputingManager(ComputingManager):
		from.utils import get_webapp_conn_string
		webapp_conn_string = get_webapp_conn_string()


		# 1) Run the container on the host (non blocking)

		# Run the container on the host (non blocking)
		if task.container.type == 'singularity':

		task.tid = task.uuid
		@@ -188,17 +186,10 @@ class RemoteComputingManager(ComputingManager):
		authstring = ''

		run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
		run_command += '/bin/bash -c \'"wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/agent_{}.py &> /dev/null && export BASE_PORT=\$(python /tmp/agent_{}.py 2> /tmp/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
		run_command += '/bin/bash -c \'"wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
		run_command += 'exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv '

		# ssh -i /rosetta/.ssh/id_rsa -4 -o StrictHostKeyChecking=no slurmclusterworker-one
		# "wget 172.21.0.2:8080/api/v1/base/agent/?task_uuid=15a4320a-88b6-4ffc-8dd0-c80f9d18b292 -O /tmp/agent_15a4320a-88b6-4ffc-8dd0-c80f9d18b292.py &> /dev/null &&
		# export BASE_PORT=\$(python /tmp/agent_15a4320a-88b6-4ffc-8dd0-c80f9d18b292.py) && export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && export SINGULARITYENV_AUTH_PASS=testpass &&
		# exec nohup singularity run --pid --writable-tmpfs --containall --cleanenv
		# docker://dregistry:5000/rosetta/metadesktop &> /tmp/15a4320a-88b6-4ffc-8dd0-c80f9d18b292.log & echo $!"


		# Set registry
		if task.container.registry == 'docker_local':
		# Get local Docker registry conn string
		@@ -210,7 +201,7 @@ class RemoteComputingManager(ComputingManager):
		else:
		raise NotImplementedError('Registry {} not supported'.format(task.container.registry))

		run_command+='{}{} &>> /tmp/{}.log & echo \$!"\''.format(registry, task.container.image, task.uuid)
		run_command+='{}{} &>> \$HOME/{}.log & echo \$!"\''.format(registry, task.container.image, task.uuid)

		else:
		raise NotImplementedError('Container {} not supported'.format(task.container.type))
		@@ -240,8 +231,8 @@ class RemoteComputingManager(ComputingManager):
		def _stop_task(self, task, **kwargs):

		# Get user keys
		if task.computing.require_user_keys:
		user_keys = Keys.objects.get(user=task.user, default=True)
		if task.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		@@ -265,8 +256,8 @@ class RemoteComputingManager(ComputingManager):
		def _get_task_log(self, task, **kwargs):

		# Get user keys
		if task.computing.require_user_keys:
		user_keys = Keys.objects.get(user=task.user, default=True)
		if task.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		@@ -275,7 +266,7 @@ class RemoteComputingManager(ComputingManager):
		user = task.computing.get_conf_param('user')

		# Stop the task remotely
		view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
		view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)

		out = os_shell(view_log_command, capture=True)
		if out.exit_code != 0:
		@@ -290,13 +281,13 @@ class SlurmComputingManager(ComputingManager):
		def _start_task(self, task, **kwargs):
		logger.debug('Starting a remote task "{}"'.format(task.computing))

		# Get computing host #Key Error ATM
		# Get computing host
		host = task.computing.get_conf_param('master')
		user = task.computing.get_conf_param('user')

		# Get user keys
		if task.computing.require_user_keys:
		user_keys = Keys.objects.get(user=task.user, default=True)
		if task.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		@@ -323,8 +314,7 @@ class SlurmComputingManager(ComputingManager):
		# Set output and error files
		sbatch_args += ' --output=\$HOME/{}.log --error=\$HOME/{}.log '.format(task.uuid, task.uuid)


		# 1) Run the container on the host (non blocking)
		# Submit the job
		if task.container.type == 'singularity':

		if not task.container.supports_dynamic_ports:
		@@ -348,7 +338,7 @@ class SlurmComputingManager(ComputingManager):
		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)

		# Double to escape for python six for shell (double times three as \\\ escapes a single slash in shell)
		# Double to escape for Pythom, six for shell (double times three as \\\ escapes a single slash in shell)

		# Set registry
		if task.container.registry == 'docker_local':
		@@ -399,8 +389,8 @@ class SlurmComputingManager(ComputingManager):
		def _stop_task(self, task, **kwargs):

		# Get user keys
		if task.computing.require_user_keys:
		user_keys = Keys.objects.get(user=task.user, default=True)
		if task.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		@@ -423,8 +413,8 @@ class SlurmComputingManager(ComputingManager):
		def _get_task_log(self, task, **kwargs):

		# Get user keys
		if task.computing.require_user_keys:
		user_keys = Keys.objects.get(user=task.user, default=True)
		if task.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+11 −11

Original line number	Diff line number	Diff line
		from django.core.management.base import BaseCommand
		from django.contrib.auth.models import User
		from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf, Keys
		from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf, KeyPair

		class Command(BaseCommand):
		help = 'Adds the admin superuser with \'a\' password.'
		@@ -34,7 +34,7 @@ class Command(BaseCommand):

		# Create default keys
		print('Creating testuser defualt keys')
		Keys.objects.create(user = testuser,
		KeyPair.objects.create(user = testuser,
		default = True,
		private_key_file = '/rosetta/.ssh/id_rsa',
		public_key_file = '/rosetta/.ssh/id_rsa.pub')
		@@ -112,9 +112,9 @@ class Command(BaseCommand):
		Computing.objects.create(user = None,
		name = 'Local',
		type = 'local',
		require_sys_conf = False,
		require_user_conf = False,
		require_user_keys = False)
		requires_sys_conf = False,
		requires_user_conf = False,
		requires_user_keys = False)


		#==============================
		@@ -123,9 +123,9 @@ class Command(BaseCommand):
		demo_remote_auth_computing = Computing.objects.create(user = None,
		name = 'Demo remote',
		type = 'remote',
		require_sys_conf = True,
		require_user_conf = True,
		require_user_keys = True)
		requires_sys_conf = True,
		requires_user_conf = True,
		requires_user_keys = True)

		ComputingSysConf.objects.create(computing = demo_remote_auth_computing,
		data = {'host': 'slurmclusterworker-one'})
		@@ -141,9 +141,9 @@ class Command(BaseCommand):
		demo_slurm_computing = Computing.objects.create(user = None,
		name = 'Demo Slurm',
		type = 'slurm',
		require_sys_conf = True,
		require_user_conf = True,
		require_user_keys = True)
		requires_sys_conf = True,
		requires_user_conf = True,
		requires_user_keys = True)

		# Create demo slurm sys computing conf
		ComputingSysConf.objects.create(computing = demo_slurm_computing,

services/webapp/code/rosetta/core_app/models.py

+30 −11

Original line number	Diff line number	Diff line
		@@ -3,7 +3,7 @@ from django.conf import settings
		from django.db import models
		from django.contrib.auth.models import User
		from django.utils import timezone
		from .utils import os_shell
		from .utils import os_shell, color_map, hash_string_to_int

		if 'sqlite' in settings.DATABASES['default']['ENGINE']:
		from .fields import JSONField
		@@ -96,7 +96,11 @@ class Container(models.Model):
		def id(self):
		return str(self.uuid).split('-')[0]


		@ property
		def color(self):
		string_int_hash = hash_string_to_int(self.name + self.type + self.image)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]

		#=========================
		# Computing resources
		@@ -111,16 +115,19 @@ class Computing(models.Model):
		name = models.CharField('Computing Name', max_length=255, blank=False, null=False)
		type = models.CharField('Computing Type', max_length=255, blank=False, null=False)

		require_sys_conf = models.BooleanField(default=False)
		require_user_conf = models.BooleanField(default=False)
		require_user_keys = models.BooleanField(default=False)
		requires_sys_conf = models.BooleanField(default=False)
		requires_user_conf = models.BooleanField(default=False)
		requires_user_keys = models.BooleanField(default=False)

		supports_docker = models.BooleanField(default=False)
		supports_singularity = models.BooleanField(default=False)


		def __str__(self):
		if self.user:
		return str('Computing Resource "{}" of user "{}"'.format(self.name, self.user))
		return str('Computing "{}" of user "{}"'.format(self.name, self.user))
		else:
		return str('Computing Resource "{}"'.format(self.name))
		return str('Computing "{}"'.format(self.name))


		@property
		@@ -172,6 +179,12 @@ class Computing(models.Model):
		ComputingManager = getattr(computing_managers, '{}ComputingManager'.format(self.type.title()))
		return ComputingManager()

		@ property
		def color(self):
		string_int_hash = hash_string_to_int(self.name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]



		class ComputingSysConf(models.Model):
		@@ -201,7 +214,7 @@ class ComputingUserConf(models.Model):

		@property
		def id(self):
		return str('Computing sys conf for {} with id "{}" of user "{}"'.format(self.computing, self.id, self.user))
		return str('Computing user conf for {} with id "{}" of user "{}"'.format(self.computing, self.id, self.user))



		@@ -274,13 +287,19 @@ class Task(models.Model):
		def __str__(self):
		return str('Task "{}" of user "{}" running on "{}" in status "{}" created at "{}"'.format(self.name, self.user, self.computing, self.status, self.created))

		@ property
		def color(self):
		string_int_hash = hash_string_to_int(self.name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]



		#=========================
		# Keys
		# KeyPair
		#=========================

		class Keys(models.Model):
		class KeyPair(models.Model):

		uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
		user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=False)
		@@ -292,7 +311,7 @@ class Keys(models.Model):


		def __str__(self):
		return str('Keys with id "{}" of user "{}"'.format(self.id, self.user))
		return str('KeyPair with id "{}" of user "{}"'.format(self.id, self.user))


		@property

services/webapp/code/rosetta/core_app/static/css/stylish-portfolio.css

+29 −8

Original line number	Diff line number	Diff line
		@@ -13,9 +13,12 @@ body {
		font-size: 1.0em;
		}

		/*body {
		font-family: "Source Sans Pro","Helvetica Neue",Helvetica,Arial,sans-serif;
		}*/
		body {
		/font-family: "Source Sans Pro","Helvetica Neue",Helvetica,Arial,sans-serif;/
		/color: #585858/
		color: #484848

		}


		.form-control {
		@@ -54,12 +57,17 @@ body {

		/* Custom Button Styles */

		.btn-dark {
		.btn-dark, .btn-dark-disabled {
		border-radius: 0;
		color: #fff;
		background-color: rgba(0,0,0,0.4);
		}

		.btn-disabled {
		border-radius: 0;
		color: #f5f5f5;
		background-color: #c8c8c8;
		}

		.btn-blue {
		border-radius: 0;
		@@ -70,8 +78,24 @@ body {

		}

		.btn-connect {
		border-radius: 0;
		color: #fff;
		background-color: rgba(0,95,144,0.8);
		}

		.btn-connect:hover {
		color: #fff;
		background-color: rgba(0,95,144,1.0);
		}

		.btn-disabled:hover {
		color: #f5f5f5;
		}

		.btn-dark-disabled:hover {
		color: #fff;
		}

		.btn-dark:hover,
		.btn-dark:focus,
		@@ -431,9 +455,6 @@ th, td {
		font-weight:700;
		}




		.centerbox-error-notfilled {
		background-color: #ffffff;
		text-align: center;
		@@ -522,7 +543,7 @@ th, td {
		/* Dashboard */
		/-------------------/
		.dashboard table {
		/border: 1px #c0c0c0 solid;/
		border: 1px #e0e0e0 solid;
		background: #F8F8F8;
		padding: 10px;
		}