Refactored the container image attributes (tag, os, arch) and made os and arch... (733e1765) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/computing_managers.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -111,7 +111,7 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
		#run_command += ' -v {}/user-{}:/data'.format(settings.LOCAL_USER_DATA_DIR, task.user.id)

		# Host name, image entry command
		run_command += ' -h task-{} -d -t {}/{}:{}'.format(task.uuid, task.container.registry, task.container.image, task.container.tag)
		run_command += ' -h task-{} -d -t {}/{}:{}'.format(task.uuid, task.container.registry, task.container.image_name, task.container.image_tag)

		# Debug
		logger.debug('Running new task with command="{}"'.format(run_command))
		@@ -238,7 +238,7 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)

		# Container part
		run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image, task.container.tag, task.uuid)
		run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid)


		else:
		@@ -377,7 +377,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)

		# Double to escape for Python, six for shell (double times three as \\\ escapes a single slash in shell)
		run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image, task.container.tag, task.uuid, task.uuid, sbatch_args, task.uuid)
		run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid, task.uuid, sbatch_args, task.uuid)

		else:
		raise NotImplementedError('Default container runtime "{}" not supported'.format(task.computing.default_container_runtime))

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+53 −51

Original line number	Diff line number	Diff line
		@@ -132,10 +132,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
		name = 'Minimal Desktop ',
		description = 'A minimal desktop environment providing basic window management functionalities and a terminal.',
		registry = 'docker.io',
		image = 'sarusso/minimaldesktop',
		tag = 'v0.2.0',
		arch = 'amd64',
		os = 'linux',
		image_name = 'sarusso/minimaldesktop',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8590',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -147,10 +147,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
		name = 'Basic Desktop',
		description = 'A basic desktop environment. Provides a terminal, a file manager, a web browser and other generic applications.',
		registry = 'docker.io',
		image = 'sarusso/basicdesktop',
		tag = 'v0.2.0',
		arch = 'amd64',
		os = 'linux',
		image_name = 'sarusso/basicdesktop',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8590',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -164,10 +164,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
		name = 'Jupyter Notebook',
		description = 'A Jupyter Notebook server',
		registry = 'docker.io',
		image = 'sarusso/jupyternotebook',
		tag = 'v0.2.0',
		arch = 'amd64',
		os = 'linux',
		image_name = 'sarusso/jupyternotebook',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -175,29 +175,32 @@ to provide help, news and informations on your deployment. Or you can just ignor
		supports_interface_auth = True,
		interface_auth_user = None)

		# Official Jupyter Lab
		# Official Jupyter containers
		for tag in ['lab-3.2.2', 'lab-3.1.17']:

		Container.objects.create(user = None,
		name = 'Jupyter Lab',
		description = 'The official Jupyter Lab. The Scipy variant, which includes popular packages from the scientific Python ecosystem.',
		name = 'Jupyter Data Science Lab',
		description = 'The official Jupyter Lab. The Data Science variant, which includes libraries for data analysis from the Julia, Python, and R communities.',
		registry = 'docker.io',
		image = 'jupyter/scipy-notebook',
		tag = 'lab-3.2.2',
		arch = 'amd64,arm64',
		os = 'linux',
		image_name = 'jupyter/scipy-notebook',
		image_tag = tag,
		image_arch = None,
		image_os = None,
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = True,
		supports_interface_auth = True)

		for arch in ['amd64', 'arm64']:
		Container.objects.create(user = None,
		name = 'Jupyter Lab',
		description = 'The official Jupyter Lab. Includes popular packages from the scientific Python ecosystem.',
		description = 'The official Jupyter Lab. The Scipy variant, which includes popular packages from the scientific Python ecosystem.',
		registry = 'docker.io',
		image = 'jupyter/scipy-notebook',
		tag = 'lab-3.1.17',
		arch = 'amd64,arm64',
		os = 'linux',
		image_name = 'jupyter/scipy-notebook',
		image_tag = tag,
		image_arch = arch,
		image_os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		@@ -205,16 +208,15 @@ to provide help, news and informations on your deployment. Or you can just ignor
		supports_interface_auth = True)



		# SSH server
		Container.objects.create(user = None,
		name = 'SSH server',
		description = 'An SSH server supporting X forwarding as well.',
		registry = 'docker.io',
		image = 'sarusso/ssh',
		tag = 'v0.2.0',
		arch = 'amd64',
		os = 'linux',
		image_name = 'sarusso/ssh',
		image_tag = 'v0.2.0',
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '22',
		interface_protocol = 'ssh',
		interface_transport = 'tcp/ip',

services/webapp/code/rosetta/core_app/migrations/0025_auto_20211122_1332.py

0 → 100644

+38 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-11-22 13:32

		from django.db import migrations, models


		class Migration(migrations.Migration):

		dependencies = [
		('core_app', '0024_computing_emulated_archs'),
		]

		operations = [
		migrations.RenameField(
		model_name='container',
		old_name='arch',
		new_name='image_arch',
		),
		migrations.RenameField(
		model_name='container',
		old_name='image',
		new_name='image_name',
		),
		migrations.RenameField(
		model_name='container',
		old_name='os',
		new_name='image_os',
		),
		migrations.RenameField(
		model_name='container',
		old_name='tag',
		new_name='image_tag',
		),
		migrations.AddField(
		model_name='container',
		name='image_digest',
		field=models.CharField(blank=True, max_length=96, null=True, verbose_name='SHA 256 digest'),
		),
		]

services/webapp/code/rosetta/core_app/migrations/0026_auto_20211123_0037.py

0 → 100644

+23 −0

Original line number	Diff line number	Diff line
		# Generated by Django 2.2.1 on 2021-11-23 00:37

		from django.db import migrations, models


		class Migration(migrations.Migration):

		dependencies = [
		('core_app', '0025_auto_20211122_1332'),
		]

		operations = [
		migrations.AlterField(
		model_name='container',
		name='image_arch',
		field=models.CharField(blank=True, max_length=36, null=True, verbose_name='Architecture'),
		),
		migrations.AlterField(
		model_name='container',
		name='image_os',
		field=models.CharField(blank=True, max_length=36, null=True, verbose_name='Operating system'),
		),
		]

services/webapp/code/rosetta/core_app/models.py

+28 −10

Original line number	Diff line number	Diff line
		import uuid
		import json
		import base64
		from django.conf import settings
		from django.db import models
		from django.contrib.auth.models import User, Group
		@@ -110,19 +111,22 @@ class Container(models.Model):
		group = models.ForeignKey(Group, related_name='containers', on_delete=models.CASCADE, blank=True, null=True)
		# If a container has no group, it will be available to anyone. Can be created, edited and deleted only by admins.


		# Generic attributes
		name = models.CharField('Name', max_length=255, blank=False, null=False)
		description = models.TextField('Description', blank=True, null=True)

		# Registry-related attributes
		# Registry
		registry = models.CharField('Registry', max_length=255, blank=False, null=False)
		image = models.CharField('Image', max_length=255, blank=False, null=False)
		tag = models.CharField('Tag', max_length=255, blank=False, null=False, default='latest')

		# Platform-related
		arch = models.CharField('Architecture', max_length=36, blank=False, null=False, default='x86_64')
		os = models.CharField('Operating system', max_length=36, blank=False, null=False, default='linux')
		# Image name
		image_name = models.CharField('Image', max_length=255, blank=False, null=False)

		# Image identifiers
		image_tag = models.CharField('Tag', max_length=255, blank=True, null=True, default='latest')
		image_arch = models.CharField('Architecture', max_length=36, blank=True, null=True)
		image_os = models.CharField('Operating system', max_length=36, blank=True, null=True)
		# -- OR --
		image_digest = models.CharField('SHA 256 digest', max_length=96, blank=True, null=True)

		# TODO: do we want more control with respect to kernel, CPUs, instruction sets?
		# requires = i.e. kernel > 3, intel, AVX2
		@@ -142,15 +146,29 @@ class Container(models.Model):

		def __str__(self):
		user_str = self.user.email if self.user else None
		return str('Container "{}" of user "{}" with image "{}" and tag "{}" on registry "{}" '.format(self.name, user_str, self.image, self.tag, self.registry))
		return str('Container "{}" of user "{}" with image name "{}" and image tag "{}" on registry "{}" '.format(self.name, user_str, self.image_name, self.image_tag, self.registry))

		def save(self, args, *kwargs):
		# Check that digest starts with sha256:
		if self.image_digest and not self.image_digest.startswith('sha256:'):
		raise ValueError('The digest field must start with "sha256:"')

		super(Container, self).save(args, *kwargs)

		@property
		def family_id(self):
		return base64.b64encode('{}\t{}\t{}'.format(self.name, self.registry, self.image_name).encode('utf8')).decode('utf8')


		@property
		def color(self):
		string_int_hash = hash_string_to_int(self.name + self.registry + self.image)
		string_int_hash = hash_string_to_int(self.name + self.registry + self.image_name)
		color_map_index = string_int_hash % len(color_map)
		return color_map[color_map_index]




		#=========================
		# Computing resources
		#=========================