Merge branch 'feature/computing_resource_refactoring' into develop (90cee362) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/api.py

+15 −11

Original line number	Diff line number	Diff line
		@@ -372,13 +372,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		user_keys = KeyPair.objects.get(user=user, default=True)

		# Get computing host
		computing_host = computing.get_conf_param('host')
		computing_host = computing.conf.get('host')

		# Trick for handling Slurm.. TODO: fix me!
		if not computing_host:
		computing_host = computing.get_conf_param('master')
		computing_host = computing.conf.get('master')

		computing_user = computing.get_conf_param('user')
		computing_user = computing.conf.get('user')

		if not computing_host:
		raise Exception('No computing host?!')
		@@ -404,13 +404,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		user_keys = KeyPair.objects.get(user=user, default=True)

		# Get computing host
		computing_host = computing.get_conf_param('host')
		computing_host = computing.conf.get('host')

		# Trick for handling Slurm.. TODO: fix me!
		if not computing_host:
		computing_host = computing.get_conf_param('master')
		computing_host = computing.conf.get('master')

		computing_user = computing.get_conf_param('user')
		computing_user = computing.conf.get('user')

		if not computing_host:
		raise Exception('No computing host?!')
		@@ -457,7 +457,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		computing = computing[0]

		# Attach user conf in any
		computing.attach_user_conf_data(request.user)
		computing.attach_user_conf(request.user)

		return computing

		@@ -711,8 +711,12 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):

		for computing in computings:

		# For now, we only support SSH-based computing resources
		if not 'ssh' in computing.access_method:
		continue

		# Attach user conf in any
		computing.attach_user_conf_data(request.user)
		computing.attach_user_conf(request.user)

		data['data'].append({
		'id': '/{}/'.format(computing.name),
		@@ -733,9 +737,9 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		# TODO: we can remove this and just always filter agains bind probably...
		if len(path.split('/')) == 3:
		if computing.user != request.user:
		binds = computing.get_conf_param('binds', from_sys_only=True )
		binds = computing.sys_conf.get('binds')
		else:
		binds = computing.get_conf_param('binds')
		binds = computing.conf.get('binds')

		if binds:
		binds = binds.split(',')

services/webapp/code/rosetta/core_app/computing_managers.py

+216 −196

File changed.

Preview size limit exceeded, changes collapsed.

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+20 −17

Original line number	Diff line number	Diff line
		@@ -209,11 +209,12 @@ class Command(BaseCommand):
		print('Creating demo computing resources containers...')

		#==============================
		# Local remote computing
		# Demo Internal computing
		#==============================
		Computing.objects.create(user = None,
		name = 'Local',
		type = 'local',
		name = 'Demo Internal',
		type = 'singlenode',
		access_method = 'internal',
		requires_sys_conf = False,
		requires_user_conf = False,
		requires_user_keys = False,
		@@ -222,32 +223,34 @@ class Command(BaseCommand):


		#==============================
		# Demo remote computing
		# Demo Single Node computing
		#==============================
		demo_remote_auth_computing = Computing.objects.create(user = None,
		name = 'Demo remote',
		type = 'remote',
		demo_singlenode_computing = Computing.objects.create(user = None,
		name = 'Demo Single Node',
		type = 'singlenode',
		access_method = 'ssh',
		requires_sys_conf = True,
		requires_user_conf = True,
		requires_user_keys = True,
		supports_docker = True,
		supports_singularity = True)

		ComputingSysConf.objects.create(computing = demo_remote_auth_computing,
		ComputingSysConf.objects.create(computing = demo_singlenode_computing,
		data = {'host': 'slurmclusterworker-one',
		'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})

		ComputingUserConf.objects.create(user = testuser,
		computing = demo_remote_auth_computing,
		computing = demo_singlenode_computing,
		data = {'user': 'slurmtestuser'})


		#==============================
		# Demo Slurm computing
		# Demo Cluster computing
		#==============================
		demo_slurm_computing = Computing.objects.create(user = None,
		name = 'Demo Slurm',
		type = 'slurm',
		name = 'Demo Cluster',
		type = 'cluster',
		access_method = 'slurm+ssh',
		requires_sys_conf = True,
		requires_user_conf = True,
		requires_user_keys = True,

services/webapp/code/rosetta/core_app/migrations/0004_auto_20210408_1041.py

0 → 100644

+30 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-04-08 10:41

		from django.db import migrations, models
		import django.db.models.deletion


		class Migration(migrations.Migration):

		dependencies = [
		('core_app', '0003_text'),
		]

		operations = [
		migrations.AddField(
		model_name='computing',
		name='access_method',
		field=models.CharField(default='NA', max_length=255, verbose_name='Computing Access method'),
		preserve_default=False,
		),
		migrations.AlterField(
		model_name='computingsysconf',
		name='computing',
		field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_sys_conf', to='core_app.Computing'),
		),
		migrations.AlterField(
		model_name='computinguserconf',
		name='computing',
		field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='related_user_conf', to='core_app.Computing'),
		),
		]

services/webapp/code/rosetta/core_app/models.py

+82 −55

Original line number	Diff line number	Diff line
		@@ -5,6 +5,7 @@ from django.db import models
		from django.contrib.auth.models import User
		from django.utils import timezone
		from .utils import os_shell, color_map, hash_string_to_int
		from .exceptions import ConsistencyException

		if 'sqlite' in settings.DATABASES['default']['ENGINE']:
		from .fields import JSONField
		@@ -138,6 +139,7 @@ class Computing(models.Model):

		name = models.CharField('Computing Name', max_length=255, blank=False, null=False)
		type = models.CharField('Computing Type', max_length=255, blank=False, null=False)
		access_method = models.CharField('Computing Access method', max_length=255, blank=False, null=False)

		requires_sys_conf = models.BooleanField(default=False)
		requires_user_conf = models.BooleanField(default=False)
		@@ -146,50 +148,72 @@ class Computing(models.Model):
		supports_docker = models.BooleanField(default=False)
		supports_singularity = models.BooleanField(default=False)

		@property
		def type_str(self):
		# TODO: improve me?
		if self.type == 'cluster':
		return 'Cluster'
		elif self.type == 'singlenode':
		return 'Single Node'
		else:
		raise ConsistencyException('Unknown computing resource type "{}"'.format(self.type))

		@property
		def access_method_str(self):
		# TODO: improve me?
		access_method = self.access_method
		access_method = access_method.replace('ssh', 'SSH')
		access_method = access_method.replace('slurm', 'Slurm')
		return access_method

		class Meta:
		ordering = ['name']


		def __str__(self):
		if self.user:
		return str('Computing "{}" of user "{}"'.format(self.name, self.user))
		else:
		return str('Computing "{}"'.format(self.name))


		@property
		def id(self):
		return str(self.uuid).split('-')[0]


		@property
		def sys_conf_data(self):
		try:
		return ComputingSysConf.objects.get(computing=self).data
		except ComputingSysConf.DoesNotExist:
		return None

		def color(self):
		string_int_hash = hash_string_to_int(self.name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]

		@property
		def sys_conf_data_json(self):
		return json.dumps(self.sys_conf_data)

		#=======================
		# Computing manager
		#=======================

		@property
		def user_conf_data(self):
		def manager(self):
		from . import computing_managers

		# Instantiate the computing manager based on type (if not already done)
		try:
		return self._user_conf_data
		return self._manager
		except AttributeError:
		raise AttributeError('User conf data is not yet attached, please attach it before accessing.')

		if self.type == 'cluster' and self.access_method == 'slurm+ssh':
		self._manager = computing_managers.SlurmSSHClusterComputingManager(self)
		elif self.type == 'singlenode' and self.access_method == 'ssh':
		self._manager = computing_managers.SSHSingleNodeComputingManager(self)
		elif self.type == 'singlenode' and self.access_method == 'internal':
		self._manager = computing_managers.InternalSingleNodeComputingManager(self)
		else:
		raise ConsistencyException('Don\'t know how to instantiate a computing manager for computing resource of type "{}" and access mode "{}"'.format(self.type, self.access_method))
		return self._manager

		@property
		def user_conf_data_json(self):
		return json.dumps(self.user_conf_data)

		#=======================
		# Sys & user conf
		#=======================

		def attach_user_conf_data(self, user):
		def attach_user_conf(self, user):
		if self.user and self.user != user:
		raise Exception('Cannot attach a conf data for another user (my user="{}", another user="{}"'.format(self.user, user))
		try:
		@@ -197,47 +221,50 @@ class Computing(models.Model):
		except ComputingUserConf.DoesNotExist:
		self._user_conf_data = None

		@property
		def sys_conf(self):
		return self.related_sys_conf.get().data

		def get_conf_param(self, param, from_sys_only=False):
		try:
		param_value = self.sys_conf_data[param]
		except (TypeError, KeyError):
		if not from_sys_only:
		@property
		def user_conf(self):
		try:
		param_value = self.user_conf_data[param]
		except (TypeError, KeyError):
		return None
		else:
		return None
		return param_value
		return self._user_conf_data
		except AttributeError:
		raise ConsistencyException('User conf has to been attached, cannot proceed.')

		@property
		def conf_params(self):
		class ConfParams():
		def __init__(self, computing):
		self.computing = computing
		def __getitem__(self, key):
		return self.computing.get_conf_param(key)
		return ConfParams(self)
		def sys_conf_as_json(self):
		return json.dumps(self.sys_conf)

		@property
		def manager(self):
		from . import computing_managers
		ComputingManager = getattr(computing_managers, '{}ComputingManager'.format(self.type.title()))
		return ComputingManager()
		def user_conf_as_json(self):
		return json.dumps(self.user_conf)

		@property
		def color(self):
		string_int_hash = hash_string_to_int(self.name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]
		def conf(self):

		if not self.requires_user_conf:
		conf_tmp = self.sys_conf
		else:
		try:
		# Copy the conf or the original user conf will be affected by the overwrite below
		conf_tmp = {key:value for key, value in self._user_conf_data.items()}
		except AttributeError:
		raise ConsistencyException('User conf has not been attached, cannot proceed.')

		# Now add (overwrite) with the sys conf
		sys_conf = self.sys_conf
		for key in sys_conf:
		conf_tmp[key] = sys_conf[key]

		return conf_tmp



		class ComputingSysConf(models.Model):

		uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
		computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
		computing = models.ForeignKey(Computing, related_name='related_sys_conf', on_delete=models.CASCADE)
		data = JSONField(blank=True, null=True)


		@@ -255,7 +282,7 @@ class ComputingUserConf(models.Model):

		uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
		user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)
		computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)
		computing = models.ForeignKey(Computing, related_name='related_user_conf', on_delete=models.CASCADE)
		data = JSONField(blank=True, null=True)

		@property