Added the manager and conf properties to the Computing class. Global computing... (b044fa68) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/api.py

+10 −10

Original line number	Diff line number	Diff line
		@@ -372,13 +372,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		user_keys = KeyPair.objects.get(user=user, default=True)

		# Get computing host
		computing_host = computing.get_conf_param('host')
		computing_host = computing.conf.get('host')

		# Trick for handling Slurm.. TODO: fix me!
		if not computing_host:
		computing_host = computing.get_conf_param('master')
		computing_host = computing.conf.get('master')

		computing_user = computing.get_conf_param('user')
		computing_user = computing.conf.get('user')

		if not computing_host:
		raise Exception('No computing host?!')
		@@ -404,13 +404,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		user_keys = KeyPair.objects.get(user=user, default=True)

		# Get computing host
		computing_host = computing.get_conf_param('host')
		computing_host = computing.conf.get('host')

		# Trick for handling Slurm.. TODO: fix me!
		if not computing_host:
		computing_host = computing.get_conf_param('master')
		computing_host = computing.conf.get('master')

		computing_user = computing.get_conf_param('user')
		computing_user = computing.conf.get('user')

		if not computing_host:
		raise Exception('No computing host?!')
		@@ -457,7 +457,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		computing = computing[0]

		# Attach user conf in any
		computing.attach_user_conf_data(request.user)
		computing.attach_user_conf(request.user)

		return computing

		@@ -712,7 +712,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		for computing in computings:

		# Attach user conf in any
		computing.attach_user_conf_data(request.user)
		computing.attach_user_conf(request.user)

		data['data'].append({
		'id': '/{}/'.format(computing.name),
		@@ -733,9 +733,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		# TODO: we can remove this and just always filter agains bind probably...
		if len(path.split('/')) == 3:
		if computing.user != request.user:
		binds = computing.get_conf_param('binds', from_sys_only=True )
		binds = computing.sys_conf.get('binds')
		else:
		binds = computing.get_conf_param('binds')
		binds = computing.conf.get('binds')

		if binds:
		binds = binds.split(',')

services/webapp/code/rosetta/core_app/computing_managers.py

+49 −46

Original line number	Diff line number	Diff line
		@@ -10,6 +10,9 @@ logger = logging.getLogger(__name__)

		class ComputingManager(object):

		def __init__(self, computing):
		self.computing = computing

		def start_task(self, task, **kwargs):

		# Check for run task logic implementation
		@@ -161,14 +164,14 @@ class LocalComputingManager(ComputingManager):
		class RemoteComputingManager(ComputingManager):

		def _start_task(self, task, **kwargs):
		logger.debug('Starting a remote task "{}"'.format(task.computing))
		logger.debug('Starting a remote task "{}"'.format(self.computing))

		# Get computing host
		host = task.computing.get_conf_param('host')
		user = task.computing.get_conf_param('user')
		host = self.computing.conf.get('host')
		user = self.computing.conf.get('user')

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
		@@ -190,10 +193,10 @@ class RemoteComputingManager(ComputingManager):
		authstring = ''

		# Set binds, only from sys config if the resource is not owned by the user
		if task.computing.user != task.user:
		binds = task.computing.get_conf_param('binds', from_sys_only=True )
		if self.computing.user != task.user:
		binds = self.computing.sys_conf.get('binds')
		else:
		binds = task.computing.get_conf_param('binds')
		binds = self.computing.conf.get('binds')
		if not binds:
		binds = ''
		else:
		@@ -253,14 +256,14 @@ class RemoteComputingManager(ComputingManager):
		def _stop_task(self, task, **kwargs):

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		# Get computing host
		host = task.computing.get_conf_param('host')
		user = task.computing.get_conf_param('user')
		host = self.computing.conf.get('host')
		user = self.computing.conf.get('user')

		# Stop the task remotely
		stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)
		@@ -277,14 +280,14 @@ class RemoteComputingManager(ComputingManager):
		def _get_task_log(self, task, **kwargs):

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		# Get computing host
		host = task.computing.get_conf_param('host')
		user = task.computing.get_conf_param('user')
		host = self.computing.conf.get('host')
		user = self.computing.conf.get('user')

		# View log remotely
		view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
		@@ -300,14 +303,14 @@ class RemoteComputingManager(ComputingManager):
		class SlurmComputingManager(ComputingManager):

		def _start_task(self, task, **kwargs):
		logger.debug('Starting a remote task "{}"'.format(task.computing))
		logger.debug('Starting a remote task "{}"'.format(self.computing))

		# Get computing host
		host = task.computing.get_conf_param('master')
		user = task.computing.get_conf_param('user')
		host = self.computing.conf.get('master')
		user = self.computing.conf.get('user')

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
		@@ -349,10 +352,10 @@ class SlurmComputingManager(ComputingManager):
		authstring = ''

		# Set binds, only from sys config if the resource is not owned by the user
		if task.computing.user != task.user:
		binds = task.computing.get_conf_param('binds', from_sys_only=True )
		if self.computing.user != task.user:
		binds = self.computing.sys_conf.get('binds')
		else:
		binds = task.computing.get_conf_param('binds')
		binds = self.computing.conf.get('binds')
		if not binds:
		binds = ''
		else:
		@@ -422,14 +425,14 @@ class SlurmComputingManager(ComputingManager):
		def _stop_task(self, task, **kwargs):

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		# Get computing host
		host = task.computing.get_conf_param('master')
		user = task.computing.get_conf_param('user')
		host = self.computing.conf.get('master')
		user = self.computing.conf.get('user')

		# Stop the task remotely
		stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)
		@@ -445,14 +448,14 @@ class SlurmComputingManager(ComputingManager):
		def _get_task_log(self, task, **kwargs):

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		# Get computing host
		host = task.computing.get_conf_param('master')
		user = task.computing.get_conf_param('user')
		host = self.computing.conf.get('master')
		user = self.computing.conf.get('user')

		# View log remotely
		view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)
		@@ -468,20 +471,20 @@ class SlurmComputingManager(ComputingManager):
		class RemotehopComputingManager(ComputingManager):

		def _start_task(self, task, **kwargs):
		logger.debug('Starting a remote task "{}"'.format(task.computing))
		logger.debug('Starting a remote task "{}"'.format(self.computing))

		# Get computing params
		first_host = task.computing.get_conf_param('first_host')
		first_user = task.computing.get_conf_param('first_user')
		second_host = task.computing.get_conf_param('second_host')
		second_user = task.computing.get_conf_param('second_user')
		setup_command = task.computing.get_conf_param('setup_command')
		first_host = self.computing.conf.get('first_host')
		first_user = self.computing.conf.get('first_user')
		second_host = self.computing.conf.get('second_host')
		second_user = self.computing.conf.get('second_user')
		setup_command = self.computing.conf.get('setup_command')

		# TODO: De hard-code
		use_agent = False

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')
		@@ -503,10 +506,10 @@ class RemotehopComputingManager(ComputingManager):
		authstring = ''

		# Set binds, only from sys config if the resource is not owned by the user
		if task.computing.user != task.user:
		binds = task.computing.get_conf_param('binds', from_sys_only=True )
		if self.computing.user != task.user:
		binds = self.computing.sys_conf.get('binds')
		else:
		binds = task.computing.get_conf_param('binds')
		binds = self.computing.conf.get('binds')
		if not binds:
		binds = ''
		else:
		@@ -581,16 +584,16 @@ class RemotehopComputingManager(ComputingManager):
		def _stop_task(self, task, **kwargs):

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		# Get computing params
		first_host = task.computing.get_conf_param('first_host')
		first_user = task.computing.get_conf_param('first_user')
		second_host = task.computing.get_conf_param('second_host')
		second_user = task.computing.get_conf_param('second_user')
		first_host = self.computing.conf.get('first_host')
		first_user = self.computing.conf.get('first_user')
		second_host = self.computing.conf.get('second_host')
		second_user = self.computing.conf.get('second_user')

		# Stop the task remotely
		stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
		@@ -610,16 +613,16 @@ class RemotehopComputingManager(ComputingManager):
		def _get_task_log(self, task, **kwargs):

		# Get user keys
		if task.computing.requires_user_keys:
		if self.computing.requires_user_keys:
		user_keys = KeyPair.objects.get(user=task.user, default=True)
		else:
		raise NotImplementedError('Remote tasks not requiring keys are not yet supported')

		# Get computing params
		first_host = task.computing.get_conf_param('first_host')
		first_user = task.computing.get_conf_param('first_user')
		second_host = task.computing.get_conf_param('second_host')
		second_user = task.computing.get_conf_param('second_user')
		first_host = self.computing.conf.get('first_host')
		first_user = self.computing.conf.get('first_user')
		second_host = self.computing.conf.get('second_host')
		second_user = self.computing.conf.get('second_user')

		# View log remotely
		view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

services/webapp/code/rosetta/core_app/models.py

+64 −56

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ from django.db import models
		from django.contrib.auth.models import User
		from django.utils import timezone
		from .utils import os_shell, color_map, hash_string_to_int
		from .exceptions import ConsistencyException

		if 'sqlite' in settings.DATABASES['default']['ENGINE']:
		from .fields import JSONField
		@@ -146,50 +147,54 @@ class Computing(models.Model):
		supports_docker = models.BooleanField(default=False)
		supports_singularity = models.BooleanField(default=False)


		class Meta:
		ordering = ['name']


		def __str__(self):
		if self.user:
		return str('Computing "{}" of user "{}"'.format(self.name, self.user))
		else:
		return str('Computing "{}"'.format(self.name))


		@property
		def id(self):
		return str(self.uuid).split('-')[0]


		@property
		def sys_conf_data(self):
		try:
		return ComputingSysConf.objects.get(computing=self).data
		except ComputingSysConf.DoesNotExist:
		return None
		def color(self):
		string_int_hash = hash_string_to_int(self.name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]


		@property
		def sys_conf_data_json(self):
		return json.dumps(self.sys_conf_data)

		#=======================
		# Computing manager
		#=======================

		@property
		def user_conf_data(self):
		def manager(self):
		from . import computing_managers

		# Instantiate the computing manager based on type (if not already done)
		try:
		return self._user_conf_data
		return self._manager
		except AttributeError:
		raise AttributeError('User conf data is not yet attached, please attach it before accessing.')

		if self.type == 'local':
		self._manager = computing_managers.LocalComputingManager(self)
		elif self.type == 'remote':
		self._manager = computing_managers.RemoteComputingManager(self)
		elif self.type == 'slurm':
		self._manager = computing_managers.SlurmComputingManager(self)
		else:
		raise ConsistencyException('Don\'t know how to instantiate a computing manager for computing resource of type "{}"'.format(self.type))
		return self._manager

		@property
		def user_conf_data_json(self):
		return json.dumps(self.user_conf_data)

		#=======================
		# Sys & user conf
		#=======================

		def attach_user_conf_data(self, user):
		def attach_user_conf(self, user):
		if self.user and self.user != user:
		raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user))
		try:
		@@ -197,47 +202,50 @@ class Computing(models.Model):
		except ComputingUserConf.DoesNotExist:
		self._user_conf_data = None

		@property
		def sys_conf(self):
		return self.related_sys_conf.get().data

		def get_conf_param(self, param, from_sys_only=False):
		try:
		param_value = self.sys_conf_data[param]
		except (TypeError, KeyError):
		if not from_sys_only:
		@property
		def user_conf(self):
		try:
		param_value = self.user_conf_data[param]
		except (TypeError, KeyError):
		return None
		else:
		return None
		return param_value
		return self._user_conf_data
		except AttributeError:
		raise ConsistencyException('User conf has to been attached, cannot proceed.')

		@property
		def conf_params(self):
		class ConfParams():
		def __init__(self, computing):
		self.computing = computing
		def __getitem__(self, key):
		return self.computing.get_conf_param(key)
		return ConfParams(self)
		def sys_conf_as_json(self):
		return json.dumps(self.sys_conf)

		@property
		def manager(self):
		from . import computing_managers
		ComputingManager = getattr(computing_managers, '{}ComputingManager'.format(self.type.title()))
		return ComputingManager()
		def user_conf_as_json(self):
		return json.dumps(self.user_conf)

		@property
		def color(self):
		string_int_hash = hash_string_to_int(self.name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]
		def conf(self):

		if not self.requires_user_conf:
		conf_tmp = self.sys_conf
		else:
		try:
		# Copy the conf or the original user conf will be affected by the overwrite below
		conf_tmp = {key:value for key, value in self._user_conf_data.items()}
		except AttributeError:
		raise ConsistencyException('User conf has not been attached, cannot proceed.')

		# Now add (overwrite) with the sys conf
		sys_conf = self.sys_conf
		for key in sys_conf:
		conf_tmp[key] = sys_conf[key]

		return conf_tmp



		class ComputingSysConf(models.Model):

		uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
		computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
		computing = models.ForeignKey(Computing, related_name='related_sys_conf', on_delete=models.CASCADE)
		data = JSONField(blank=True, null=True)


		@@ -255,7 +263,7 @@ class ComputingUserConf(models.Model):

		uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
		user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
		computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
		computing = models.ForeignKey(Computing, related_name='related_user_conf', on_delete=models.CASCADE)
		data = JSONField(blank=True, null=True)

		@property

services/webapp/code/rosetta/core_app/templates/components/computing.html

+2 −2

Original line number	Diff line number	Diff line
		@@ -48,12 +48,12 @@

		<tr>
		<td><b>Sys Conf</b></td>
		<td>{{ data.computing.sys_conf_data_json }} {% if request.user.is_superuser %}  [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td>
		<td>{{ data.computing.sys_conf_as_json }} {% if request.user.is_superuser %}  [<a href="/edit_computing_conf?type=sys&computing_uuid={{ data.computing.uuid}}">Edit</a>] {% endif %}</td>
		</tr>

		<tr>
		<td><b>User Conf</b></td>
		<td>{{ data.computing.user_conf_data_json }}  [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td>
		<td>{{ data.computing.user_conf_as_json }}  [<a href="/edit_computing_conf?type=user&computing_uuid={{ data.computing.uuid}}">Edit</a>]</td>
		</tr>

services/webapp/code/rosetta/core_app/templates/create_task.html

+3 −3

Original line number	Diff line number	Diff line
		@@ -185,9 +185,9 @@
		<td><b>Computing options</b></td>
		<td>
		<table>
		<tr><td>Partition</td><td><input type="text" name="computing_partition" value="{{ data.task_computing.conf_params.default_partition }}" placeholder="" size="20" /></td></tr>
		<tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="{{ data.task_computing.conf_params.default_cpus }}" placeholder="" size="5" /></td></tr>
		<tr><td>Memory</td><td><input type="text" name="computing_memory" value="{{ data.task_computing.conf_params.default_memory }}" placeholder="" size="5" /></td></tr>
		<tr><td>Partition</td><td><input type="text" name="computing_partition" value="{{ data.task_computing.conf.default_partition }}" placeholder="" size="20" /></td></tr>
		<tr><td>Cpus</td><td><input type="text" name="computing_cpus" value="{{ data.task_computing.conf.default_cpus }}" placeholder="" size="5" /></td></tr>
		<tr><td>Memory</td><td><input type="text" name="computing_memory" value="{{ data.task_computing.conf.default_memory }}" placeholder="" size="5" /></td></tr>
		</table>
		</td>
		</tr>