Merge branch 'feature/refactor_computing_conf' into develop (8b28c6cd) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/admin.py

+1 −3

Original line number	Diff line number	Diff line
		from django.contrib import admin

		from .models import Profile, LoginToken, Task, Container, Computing, ComputingConf, ComputingUserConf, Storage, KeyPair, Text
		from .models import Profile, LoginToken, Task, Container, Computing, Storage, KeyPair, Text

		admin.site.register(Profile)
		admin.site.register(LoginToken)
		admin.site.register(Task)
		admin.site.register(Container)
		admin.site.register(Computing)
		admin.site.register(ComputingConf)
		admin.site.register(ComputingUserConf)
		admin.site.register(Storage)
		admin.site.register(KeyPair)
		admin.site.register(Text)

services/webapp/code/rosetta/core_app/api.py

+15 −6

Original line number	Diff line number	Diff line
		@@ -417,9 +417,11 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		if '$SSH_USER' in base_path_expanded:
		if storage.access_through_computing:
		computing = storage.computing
		computing.attach_user_conf(user)
		if computing.auth_mode == 'user_keys':
		base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.user_conf.get('user'))
		computing_user = user.profile.get_extra_conf('computing_user', storage.computing)
		if not computing_user:
		raise Exception('Computing resource \'{}\' user is not configured'.format(storage.computing.name))
		base_path_expanded = base_path_expanded.replace('$SSH_USER', computing_user)
		else:
		base_path_expanded = base_path_expanded.replace('$SSH_USER', computing.conf.get('user'))

		@@ -444,16 +446,23 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		except IndexError:
		computing_name = None

		# Get all the storages for this name:
		storages = Storage.objects.filter(name=storage_name, user=None)
		# Get all the storages this user has access to:
		storages = list(Storage.objects.filter(group=None, name=storage_name)) + list(Storage.objects.filter(group__user=request.user, name=storage_name))

		# Filter by computing resource name
		# Filter by computing resource name (or None)
		if computing_name:
		unfiltered_storages = storages
		storages = []
		for storage in unfiltered_storages:
		if storage.computing.name == computing_name:
		storages.append(storage)
		else:
		unfiltered_storages = storages
		storages = []
		for storage in unfiltered_storages:
		if storage.computing is None:
		storages.append(storage)


		# Check that we had at least and no more than one storage in the end
		if len(storages) == 0:
		@@ -714,7 +723,7 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		data = {'data':[]}

		# Get storages
		storages = list(Storage.objects.filter(user=None)) + list(Storage.objects.filter(user=request.user))
		storages = list(Storage.objects.filter(group=None)) + list(Storage.objects.filter(group__user=request.user))

		for storage in storages:

services/webapp/code/rosetta/core_app/computing_managers.py

+0 −6

Original line number	Diff line number	Diff line
		@@ -171,10 +171,6 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):







		class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingManager):

		def _start_task(self, task, **kwargs):
		@@ -208,7 +204,6 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
		expanded_base_path = storage.base_path
		if '$SSH_USER' in expanded_base_path:
		if storage.access_through_computing:
		self.computing.attach_user_conf(self.computing.user)
		expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
		else:
		raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')
		@@ -348,7 +343,6 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
		expanded_base_path = storage.base_path
		if '$SSH_USER' in expanded_base_path:
		if storage.access_through_computing:
		self.computing.attach_user_conf(self.computing.user)
		expanded_base_path = expanded_base_path.replace('$SSH_USER', computing_user)
		else:
		raise NotImplementedError('Accessing a storage with ssh+cli without going through its computing resource is not implemented')

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+28 −44

Original line number	Diff line number	Diff line
		from django.core.management.base import BaseCommand
		from django.contrib.auth.models import User
		from ...models import Profile, Container, Computing, ComputingConf, ComputingUserConf, Storage, KeyPair, Text
		from ...models import Profile, Container, Computing, Storage, KeyPair, Text

		class Command(BaseCommand):
		help = 'Adds the admin superuser with \'a\' password.'
		@@ -50,7 +50,7 @@ class Command(BaseCommand):
		#=====================
		# TODO: create a different pair
		try:
		testuser = KeyPair.objects.get(user=None, default=True)
		KeyPair.objects.get(user=None, default=True)
		print('Not creating default platform keys as they already exist')

		except KeyPair.DoesNotExist:
		@@ -193,8 +193,7 @@ class Command(BaseCommand):
		print('Creating demo computing resources...')

		# Demo internal computing
		Computing.objects.create(user = None,
		name = 'Demo Internal',
		Computing.objects.create(name = 'Demo Internal',
		description = 'A demo internal computing resource.',
		type = 'standalone',
		access_mode = 'internal',
		@@ -204,43 +203,30 @@ class Command(BaseCommand):


		# Demo standalone computing plus conf
		demo_singlenode_computing = Computing.objects.create(user = None,
		name = 'Demo Standalone',
		demo_singlenode_computing = Computing.objects.create(name = 'Demo Standalone',
		description = 'A demo standalone computing resource.',
		type = 'standalone',
		access_mode = 'ssh+cli',
		auth_mode = 'user_keys',
		wms = None,
		conf = {'host': 'slurmclusterworker-one'},
		container_runtimes = 'singularity')

		ComputingConf.objects.create(computing = demo_singlenode_computing,
		data = {'host': 'slurmclusterworker-one',
		'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})

		ComputingUserConf.objects.create(user = testuser,
		computing = demo_singlenode_computing,
		data = {'user': 'slurmtestuser'})

		# Add testuser extra conf for this computing resource
		testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_singlenode_computing, value= 'slurmtestuser')

		# Demo cluster computing plus conf
		demo_slurm_computing = Computing.objects.create(user = None,
		name = 'Demo Cluster',
		demo_slurm_computing = Computing.objects.create(name = 'Demo Cluster',
		description = 'A demo cluster computing resource.',
		type = 'cluster',
		access_mode = 'ssh+cli',
		auth_mode = 'user_keys',
		wms = 'slurm',
		conf = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1'},
		container_runtimes = 'singularity')

		ComputingConf.objects.create(computing = demo_slurm_computing,
		data = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1',
		'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})

		ComputingUserConf.objects.create(user = testuser,
		computing = demo_slurm_computing,
		data = {'user': 'slurmtestuser'})


		# Add testuser extra conf for this computing resource
		testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'slurmtestuser')

		#=====================
		# Storages
		@@ -267,8 +253,7 @@ class Command(BaseCommand):

		for computing in demo_computing_resources:
		# Demo shared computing plus conf
		Storage.objects.create(user = None,
		computing = computing,
		Storage.objects.create(computing = computing,
		access_through_computing = True,
		name = 'Shared',
		type = 'generic_posix',
		@@ -278,8 +263,7 @@ class Command(BaseCommand):
		bind_path = '/storages/shared')

		# Demo shared computing plus conf
		Storage.objects.create(user = None,
		computing = computing,
		Storage.objects.create(computing = computing,
		access_through_computing = True,
		name = 'Personal',
		type = 'generic_posix',

services/webapp/code/rosetta/core_app/migrations/0015_auto_20211108_1639.py

0 → 100644

+83 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-11-08 16:39

		from django.conf import settings
		import django.contrib.postgres.fields.jsonb
		from django.db import migrations, models
		import django.db.models.deletion


		class Migration(migrations.Migration):

		dependencies = [
		('auth', '0011_update_proxy_permissions'),
		('core_app', '0014_auto_20211108_1548'),
		]

		operations = [
		migrations.RenameField(
		model_name='storage',
		old_name='config',
		new_name='conf',
		),
		migrations.RemoveField(
		model_name='computing',
		name='user',
		),
		migrations.RemoveField(
		model_name='storage',
		name='user',
		),
		migrations.AddField(
		model_name='computing',
		name='conf',
		field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True),
		),
		migrations.AddField(
		model_name='computing',
		name='group',
		field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='computings', to='auth.Group'),
		),
		migrations.AddField(
		model_name='container',
		name='group',
		field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='containers', to='auth.Group'),
		),
		migrations.AddField(
		model_name='storage',
		name='group',
		field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='storages', to='auth.Group'),
		),
		migrations.AlterField(
		model_name='computinguserconf',
		name='computing',
		field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='user_confs', to='core_app.Computing'),
		),
		migrations.AlterField(
		model_name='computinguserconf',
		name='user',
		field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='computing_confs', to=settings.AUTH_USER_MODEL),
		),
		migrations.AlterField(
		model_name='container',
		name='user',
		field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='containers', to=settings.AUTH_USER_MODEL),
		),
		migrations.AlterField(
		model_name='keypair',
		name='user',
		field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='key_pairs', to=settings.AUTH_USER_MODEL),
		),
		migrations.AlterField(
		model_name='task',
		name='computing',
		field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to='core_app.Computing'),
		),
		migrations.AlterField(
		model_name='task',
		name='user',
		field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tasks', to=settings.AUTH_USER_MODEL),
		),
		migrations.DeleteModel(
		name='ComputingConf',
		),
		]