Merge branch 'feature/arch_support' into develop (f5319aba) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/admin.py

+5 −1

Original line number	Diff line number	Diff line
		@@ -2,10 +2,14 @@ from django.contrib import admin

		from .models import Profile, LoginToken, Task, Container, Computing, Storage, KeyPair, Page

		# Define a extra "ModelAdmin" for the Container model to allow "save as" to easily duplicate containers
		class ContainerForAdmin(admin.ModelAdmin):
		save_as = True

		admin.site.register(Profile)
		admin.site.register(LoginToken)
		admin.site.register(Task)
		admin.site.register(Container)
		admin.site.register(Container, ContainerForAdmin)
		admin.site.register(Computing)
		admin.site.register(Storage)
		admin.site.register(KeyPair)

services/webapp/code/rosetta/core_app/computing_managers.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -111,7 +111,7 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
		#run_command += ' -v {}/user-{}:/data'.format(settings.LOCAL_USER_DATA_DIR, task.user.id)

		# Host name, image entry command
		run_command += ' -h task-{} -d -t {}/{}:{}'.format(task.uuid, task.container.registry, task.container.image, task.container.tag)
		run_command += ' -h task-{} -d -t {}/{}:{}'.format(task.uuid, task.container.registry, task.container.image_name, task.container.image_tag)

		# Debug
		logger.debug('Running new task with command="{}"'.format(run_command))
		@@ -238,7 +238,7 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)

		# Container part
		run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image, task.container.tag, task.uuid)
		run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid)


		else:
		@@ -377,7 +377,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)

		# Double to escape for Python, six for shell (double times three as \\\ escapes a single slash in shell)
		run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image, task.container.tag, task.uuid, task.uuid, sbatch_args, task.uuid)
		run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid, task.uuid, sbatch_args, task.uuid)

		else:
		raise NotImplementedError('Default container runtime "{}" not supported'.format(task.computing.default_container_runtime))

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+60 −27

Original line number	Diff line number	Diff line
		@@ -132,10 +132,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
		name = 'Minimal Desktop',
		description = 'A minimal desktop environment providing basic window management functionalities and a terminal.',
		registry = 'docker.io',
		image = 'sarusso/minimaldesktop',
		tag = 'v0.2.0',
		arch = 'x86_64',
		os = 'linux',
		image_name = 'sarusso/minimaldesktop',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8590',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -147,10 +147,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
		name = 'Basic Desktop',
		description = 'A basic desktop environment. Provides a terminal, a file manager, a web browser and other generic applications.',
		registry = 'docker.io',
		image = 'sarusso/basicdesktop',
		tag = 'v0.2.0',
		arch = 'x86_64',
		os = 'linux',
		image_name = 'sarusso/basicdesktop',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8590',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -164,10 +164,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
		name = 'Jupyter Notebook',
		description = 'A Jupyter Notebook server',
		registry = 'docker.io',
		image = 'sarusso/jupyternotebook',
		tag = 'v0.2.0',
		arch = 'x86_64',
		os = 'linux',
		image_name = 'sarusso/jupyternotebook',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -175,15 +175,48 @@ to provide help, news and informations on your deployment. Or you can just ignor
		supports_interface_auth = True,
		interface_auth_user = None)

		# Official Jupyter containers
		for tag in ['lab-3.2.2', 'lab-3.1.17']:

		Container.objects.create(user = None,
		name = 'Jupyter Data Science Lab',
		description = 'The official Jupyter Lab. The Data Science variant, which includes libraries for data analysis from the Julia, Python, and R communities.',
		registry = 'docker.io',
		image_name = 'jupyter/scipy-notebook',
		image_tag = tag,
		image_arch = None,
		image_os = None,
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = True,
		supports_interface_auth = True)

		for arch in ['amd64', 'arm64']:
		Container.objects.create(user = None,
		name = 'Jupyter Lab',
		description = 'The official Jupyter Lab. The Scipy variant, which includes popular packages from the scientific Python ecosystem.',
		registry = 'docker.io',
		image_name = 'jupyter/scipy-notebook',
		image_tag = tag,
		image_arch = arch,
		image_os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = True,
		supports_interface_auth = True)


		# SSH server
		Container.objects.create(user = None,
		name = 'SSH server',
		description = 'An SSH server supporting X forwarding as well.',
		registry = 'docker.io',
		image = 'sarusso/ssh',
		tag = 'v0.2.0',
		arch = 'x86_64',
		os = 'linux',
		image_name = 'sarusso/ssh',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '22',
		interface_protocol = 'ssh',
		interface_transport = 'tcp/ip',
		@@ -232,7 +265,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
		access_mode = 'internal',
		auth_mode = 'internal',
		wms = None,
		container_runtimes = 'docker')
		container_runtimes = ['docker'])


		# Demo standalone computing plus conf
		@@ -243,7 +276,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
		auth_mode = 'user_keys',
		wms = None,
		conf = {'host': 'slurmclusterworker-one'},
		container_runtimes = 'singularity')
		container_runtimes = ['singularity'])

		# Add testuser extra conf for this computing resource
		testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_singlenode_computing, value= 'slurmtestuser')
		@@ -256,7 +289,7 @@ to provide help, news and informations on your deployment. Or you can just ignor
		auth_mode = 'user_keys',
		wms = 'slurm',
		conf = {'host': 'slurmclustermaster-main', 'default_partition': 'partition1'},
		container_runtimes = 'singularity')
		container_runtimes = ['singularity'])

		# Add testuser extra conf for this computing resource
		testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'slurmtestuser')

services/webapp/code/rosetta/core_app/migrations/0022_auto_20211121_1341.py

0 → 100644

+18 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-11-21 13:41

		from django.db import migrations


		class Migration(migrations.Migration):

		dependencies = [
		('core_app', '0021_container_interface_auth_user'),
		]

		operations = [
		migrations.RenameField(
		model_name='computing',
		old_name='container_runtimes',
		new_name='container_runtime',
		),
		]

services/webapp/code/rosetta/core_app/migrations/0023_auto_20211121_1342.py

0 → 100644

+24 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-11-21 13:42

		import django.contrib.postgres.fields.jsonb
		from django.db import migrations


		class Migration(migrations.Migration):

		dependencies = [
		('core_app', '0022_auto_20211121_1341'),
		]

		operations = [
		migrations.RemoveField(
		model_name='computing',
		name='container_runtime',
		),
		migrations.AddField(
		model_name='computing',
		name='container_runtimes',
		field=django.contrib.postgres.fields.jsonb.JSONField(default=['docker'], verbose_name='Container runtimes'),
		preserve_default=False,
		),
		]