Commit 733e1765 authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Refactored the container image attributes (tag, os, arch) and made os and arch...

Refactored the container image attributes (tag, os, arch) and made os and arch not mandatory. Improved navigation and added arch checks when submitting tasks.
parent 3efa2d97
Loading
Loading
Loading
Loading
+3 −3
Original line number Diff line number Diff line
@@ -111,7 +111,7 @@ class InternalSingleNodeComputingManager(SingleNodeComputingManager):
        #run_command += ' -v {}/user-{}:/data'.format(settings.LOCAL_USER_DATA_DIR, task.user.id)

        # Host name, image entry command
        run_command += ' -h task-{} -d -t {}/{}:{}'.format(task.uuid, task.container.registry, task.container.image, task.container.tag)
        run_command += ' -h task-{} -d -t {}/{}:{}'.format(task.uuid, task.container.registry, task.container.image_name, task.container.image_tag)

        # Debug
        logger.debug('Running new task with command="{}"'.format(run_command))
@@ -238,7 +238,7 @@ class SSHSingleNodeComputingManager(SingleNodeComputingManager, SSHComputingMana
            run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
            
            # Container part
            run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image, task.container.tag, task.uuid)
            run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid)
            

        else:
@@ -377,7 +377,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag
            run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --no-home --home=/home/metauser --workdir /tmp/{}_data/tmp -B/tmp/{}_data/home:/home --containall --cleanenv '.format(binds, task.uuid, task.uuid)
            
            # Double to escape for Python, six for shell (double times three as \\\ escapes a single slash in shell)
            run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image, task.container.tag, task.uuid, task.uuid, sbatch_args, task.uuid)
            run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid, task.uuid, sbatch_args, task.uuid)

        else:
            raise NotImplementedError('Default container runtime "{}" not supported'.format(task.computing.default_container_runtime))
+53 −51
Original line number Diff line number Diff line
@@ -132,10 +132,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
                                     name     = 'Minimal Desktop ',
                                     description = 'A minimal desktop environment providing basic window management functionalities and a terminal.',
                                     registry = 'docker.io',
                                     image    = 'sarusso/minimaldesktop',
                                     tag      = 'v0.2.0',
                                     arch = 'amd64',
                                     os = 'linux',
                                     image_name = 'sarusso/minimaldesktop',
                                     image_tag  = 'v0.2.0',
                                     image_arch = 'amd64',
                                     image_os   = 'linux',
                                     interface_port      = '8590',
                                     interface_protocol  = 'http',
                                     interface_transport = 'tcp/ip',
@@ -147,10 +147,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
                                     name     = 'Basic Desktop',
                                     description = 'A basic desktop environment. Provides a terminal, a file manager, a web browser and other generic applications.',
                                     registry = 'docker.io',
                                     image    = 'sarusso/basicdesktop',
                                     tag      = 'v0.2.0',
                                     arch = 'amd64',
                                     os = 'linux',
                                     image_name = 'sarusso/basicdesktop',
                                     image_tag  = 'v0.2.0',
                                     image_arch = 'amd64',
                                     image_os   = 'linux',
                                     interface_port      = '8590',
                                     interface_protocol  = 'http',
                                     interface_transport = 'tcp/ip',
@@ -164,10 +164,10 @@ to provide help, news and informations on your deployment. Or you can just ignor
                                     name     = 'Jupyter Notebook',
                                     description = 'A Jupyter Notebook server',
                                     registry = 'docker.io',
                                     image    = 'sarusso/jupyternotebook',
                                     tag      = 'v0.2.0',
                                     arch = 'amd64',
                                     os = 'linux',
                                     image_name = 'sarusso/jupyternotebook',
                                     image_tag  = 'v0.2.0',
                                     image_arch = 'amd64',
                                     image_os   = 'linux',
                                     interface_port      = '8888',
                                     interface_protocol  = 'http',
                                     interface_transport = 'tcp/ip',
@@ -175,29 +175,32 @@ to provide help, news and informations on your deployment. Or you can just ignor
                                     supports_interface_auth = True,
                                     interface_auth_user = None)

            # Official Jupyter Lab
            # Official Jupyter containers
            for tag in ['lab-3.2.2', 'lab-3.1.17']:
                
                Container.objects.create(user     = None,
                                     name     = 'Jupyter Lab',
                                     description = 'The official Jupyter Lab. The Scipy variant, which includes popular packages from the scientific Python ecosystem.',
                                         name     = 'Jupyter Data Science Lab',
                                         description = 'The official Jupyter Lab. The Data Science variant, which includes libraries for data analysis from the Julia, Python, and R communities.',
                                         registry = 'docker.io',
                                     image    = 'jupyter/scipy-notebook',
                                     tag      = 'lab-3.2.2',
                                     arch = 'amd64,arm64',
                                     os = 'linux',
                                         image_name = 'jupyter/scipy-notebook',
                                         image_tag  = tag,
                                         image_arch = None,
                                         image_os   = None,
                                         interface_port      = '8888',
                                         interface_protocol  = 'http',
                                         interface_transport = 'tcp/ip',
                                         supports_custom_interface_port = True,
                                         supports_interface_auth = True)
                
                for arch in ['amd64', 'arm64']:
                    Container.objects.create(user     = None,
                                             name     = 'Jupyter Lab',
                                     description = 'The official Jupyter Lab. Includes popular packages from the scientific Python ecosystem.',
                                             description = 'The official Jupyter Lab. The Scipy variant, which includes popular packages from the scientific Python ecosystem.',
                                             registry = 'docker.io',
                                     image    = 'jupyter/scipy-notebook',
                                     tag      = 'lab-3.1.17',
                                     arch = 'amd64,arm64',
                                     os = 'linux',
                                             image_name = 'jupyter/scipy-notebook',
                                             image_tag  = tag,
                                             image_arch = arch,
                                             image_os   = 'linux',
                                             interface_port      = '8888',
                                             interface_protocol  = 'http',
                                             interface_transport = 'tcp/ip',
@@ -205,16 +208,15 @@ to provide help, news and informations on your deployment. Or you can just ignor
                                             supports_interface_auth = True)



            # SSH server
            Container.objects.create(user     = None,
                                     name     = 'SSH server',
                                     description = 'An SSH server supporting X forwarding as well.',
                                     registry = 'docker.io',
                                     image    = 'sarusso/ssh',
                                     tag      = 'v0.2.0',
                                     arch = 'amd64',
                                     os = 'linux',
                                     image_name = 'sarusso/ssh',
                                     image_tag  = 'v0.2.0',
                                     image_arch = 'amd64',
                                     image_os   = 'linux',
                                     interface_port     = '22',
                                     interface_protocol = 'ssh',
                                     interface_transport = 'tcp/ip',
+38 −0
Original line number Diff line number Diff line
# Generated by Django 2.2.1 on 2021-11-22 13:32

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('core_app', '0024_computing_emulated_archs'),
    ]

    operations = [
        migrations.RenameField(
            model_name='container',
            old_name='arch',
            new_name='image_arch',
        ),
        migrations.RenameField(
            model_name='container',
            old_name='image',
            new_name='image_name',
        ),
        migrations.RenameField(
            model_name='container',
            old_name='os',
            new_name='image_os',
        ),
        migrations.RenameField(
            model_name='container',
            old_name='tag',
            new_name='image_tag',
        ),
        migrations.AddField(
            model_name='container',
            name='image_digest',
            field=models.CharField(blank=True, max_length=96, null=True, verbose_name='SHA 256 digest'),
        ),
    ]
+23 −0
Original line number Diff line number Diff line
# Generated by Django 2.2.1 on 2021-11-23 00:37

from django.db import migrations, models


class Migration(migrations.Migration):

    dependencies = [
        ('core_app', '0025_auto_20211122_1332'),
    ]

    operations = [
        migrations.AlterField(
            model_name='container',
            name='image_arch',
            field=models.CharField(blank=True, max_length=36, null=True, verbose_name='Architecture'),
        ),
        migrations.AlterField(
            model_name='container',
            name='image_os',
            field=models.CharField(blank=True, max_length=36, null=True, verbose_name='Operating system'),
        ),
    ]
+28 −10
Original line number Diff line number Diff line
import uuid
import json
import base64
from django.conf import settings
from django.db import models
from django.contrib.auth.models import User, Group
@@ -110,19 +111,22 @@ class Container(models.Model):
    group = models.ForeignKey(Group, related_name='containers', on_delete=models.CASCADE, blank=True, null=True)
    # If a container has no group, it will be available to anyone. Can be created, edited and deleted only by admins.


    # Generic attributes
    name        = models.CharField('Name', max_length=255, blank=False, null=False)
    description = models.TextField('Description', blank=True, null=True)
    
    # Registry-related attributes
    # Registry
    registry = models.CharField('Registry', max_length=255, blank=False, null=False)
    image    = models.CharField('Image', max_length=255, blank=False, null=False)
    tag      = models.CharField('Tag', max_length=255, blank=False, null=False, default='latest')

    # Platform-related
    arch = models.CharField('Architecture', max_length=36, blank=False, null=False, default='x86_64')
    os   = models.CharField('Operating system', max_length=36, blank=False, null=False, default='linux')
    # Image name
    image_name = models.CharField('Image', max_length=255, blank=False, null=False)
    
    # Image identifiers
    image_tag  = models.CharField('Tag', max_length=255, blank=True, null=True, default='latest')
    image_arch = models.CharField('Architecture', max_length=36, blank=True, null=True)
    image_os   = models.CharField('Operating system', max_length=36, blank=True, null=True)
    # -- OR --
    image_digest  = models.CharField('SHA 256 digest', max_length=96, blank=True, null=True)
    
    # TODO: do we want more control with respect to kernel, CPUs, instruction sets? 
    # requires = i.e. kernel > 3, intel, AVX2
@@ -142,15 +146,29 @@ class Container(models.Model):

    def __str__(self):
        user_str = self.user.email if self.user else None
        return str('Container "{}" of user "{}" with image "{}" and tag "{}" on registry "{}" '.format(self.name, user_str, self.image, self.tag, self.registry))
        return str('Container "{}" of user "{}" with image name "{}" and image tag "{}" on registry "{}" '.format(self.name, user_str, self.image_name, self.image_tag, self.registry))

    def save(self, *args, **kwargs):
        # Check that digest starts with sha256:
        if self.image_digest and not self.image_digest.startswith('sha256:'):
            raise ValueError('The digest field must start with "sha256:"')
        
        super(Container, self).save(*args, **kwargs)

    @property
    def family_id(self):
        return base64.b64encode('{}\t{}\t{}'.format(self.name, self.registry, self.image_name).encode('utf8')).decode('utf8')


    @property
    def color(self):
        string_int_hash = hash_string_to_int(self.name + self.registry + self.image)
        string_int_hash = hash_string_to_int(self.name + self.registry + self.image_name)
        color_map_index = string_int_hash % len(color_map)
        return color_map[color_map_index]




#=========================
#  Computing resources
#=========================
Loading