Commit 98ab8b36 authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Moved from container_runtimes to container_engines.

parent 6c1ae6ea

services/webapp/code/rosetta/core_app/api.py

+7 −7
Original line number Diff line number Diff line
@@ -325,19 +325,19 @@ print(port) 
            task.status = TaskStatuses.running

            task.interface_ip = task_interface_ip

            

            # Get container runtime

            container_runtime = None

            # Get container engine

            container_engine = None

            if task.computing_options:

                container_runtime = task.computing_options.get('container_runtime', None)

            if not container_runtime:

                container_runtime = task.computing.default_container_runtime

                container_engine = task.computing_options.get('container_engine', None)

            if not container_engine:

                container_engine = task.computing.default_container_engine

            

            if container_runtime=='singularity':

            if container_engine=='singularity':

                # For Singularity, set this only if the container supports custom interface ports

                if task.container.supports_custom_interface_port:

                    task.interface_port = int(task_interface_port)

            else:

                # For all other container runtimes, set it in any case

                # For all other container engines, set it in any case

                task.interface_port = int(task_interface_port)

            

            # Save the task

services/webapp/code/rosetta/core_app/computing_managers.py

+40 −40
Original line number Diff line number Diff line
@@ -188,15 +188,15 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
        from.utils import get_webapp_conn_string

        webapp_conn_string = get_webapp_conn_string()

            

        # Handle container runtime

        container_runtime = None

        # Handle container engine

        container_engine = None

        if task.computing_options:

            container_runtime = task.computing_options.get('container_runtime', None)

        if not container_runtime:

            container_runtime = task.computing.default_container_runtime

            container_engine = task.computing_options.get('container_engine', None)

        if not container_engine:

            container_engine = task.computing.default_container_engine



        # Runtime-specific part 

        if container_runtime == 'singularity':

        # engine-specific part 

        if container_engine == 'singularity':



            #if not task.container.supports_custom_interface_port:

            #     raise Exception('This task does not support dynamic port allocation and is therefore not supported using singularity on Slurm')
@@ -248,7 +248,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
            run_command+='docker://{}/{}:{} &>> /tmp/{}_data/task.log & echo \$!"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid)





        elif container_runtime in ['docker', 'podman']:

        elif container_engine in ['docker', 'podman']:



            # Set pass if any

            authstring = ''
@@ -288,20 +288,20 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
                        binds += ' -v{}:{}'.format(expanded_base_path, expanded_bind_path)

            

            # TODO: remove this hardcoding

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_runtime=='docker') else ''

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''

            

            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)

            run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid) 

            run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += '{} {} run -p \$TASK_PORT:{} {} {} '.format(prefix, container_runtime, task.container.interface_port, authstring, binds)        

            if container_runtime == 'podman':

            run_command += '{} {} run -p \$TASK_PORT:{} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, binds)        

            if container_engine == 'podman':

                run_command += '--network=private --uts=private '

            #run_command += '-d -t {}/{}:{}'.format(task.container.registry, task.container.image_name, task.container.image_tag)

            run_command += '-h task-{} -d -t {}/{}:{}'.format(task.short_uuid, task.container.registry, task.container.image_name, task.container.image_tag)

            run_command += '"\''

            

        else:

            raise NotImplementedError('Container runtime {} not supported'.format(container_runtime))

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))



        out = os_shell(run_command, capture=True)

        if out.exit_code != 0:
@@ -326,21 +326,21 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)



        # Handle container runtime

        container_runtime = None

        # Handle container engine

        container_engine = None

        if task.computing_options:

            container_runtime = task.computing_options.get('container_runtime', None)

        if not container_runtime:

            container_runtime = task.computing.default_container_runtime

            container_engine = task.computing_options.get('container_engine', None)

        if not container_engine:

            container_engine = task.computing.default_container_engine



        if container_runtime=='singularity':

        if container_engine=='singularity':

            internal_stop_command = 'kill -9 {}'.format(task.id)            

        elif container_runtime in ['docker', 'podman']:

        elif container_engine in ['docker', 'podman']:

            # TODO: remove this hardcoding

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_runtime=='docker') else ''

            internal_stop_command = '{} {} stop {} && {} {} rm {}'.format(prefix,container_runtime,task.id,prefix,container_runtime,task.id)

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''

            internal_stop_command = '{} {} stop {} && {} {} rm {}'.format(prefix,container_engine,task.id,prefix,container_engine,task.id)

        else:

            raise NotImplementedError('Container runtime {} not supported'.format(container_runtime))

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))



        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_keys.private_key_file, computing_user, computing_host, internal_stop_command)

        out = os_shell(stop_command, capture=True)
@@ -360,21 +360,21 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        

        # Handle container runtime

        container_runtime = None

        # Handle container engine

        container_engine = None

        if task.computing_options:

            container_runtime = task.computing_options.get('container_runtime', None)

        if not container_runtime:

            container_runtime = task.computing.default_container_runtime

            container_engine = task.computing_options.get('container_engine', None)

        if not container_engine:

            container_engine = task.computing.default_container_engine



        if container_runtime=='singularity':

        if container_engine=='singularity':

            internal_view_log_command = 'cat /tmp/{}_data/task.log'.format(task.uuid)            

        elif container_runtime in ['docker','podman']:

        elif container_engine in ['docker','podman']:

            # TODO: remove this hardcoding

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_runtime=='docker') else ''

            internal_view_log_command = '{} {} logs {}'.format(prefix,container_runtime,task.id)

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''

            internal_view_log_command = '{} {} logs {}'.format(prefix,container_engine,task.id)

        else:

            raise NotImplementedError('Container runtime {} not supported'.format(container_runtime))

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))

            

        # Prepare full comand

        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_keys.private_key_file, computing_user, computing_host, internal_view_log_command)
@@ -419,15 +419,15 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag 
        # Set output and error files

        sbatch_args += ' --output=\$HOME/{}.log --error=\$HOME/{}.log '.format(task.uuid, task.uuid)



        # Handle container runtime

        container_runtime = None

        # Handle container engine

        container_engine = None

        if task.computing_options:

            container_runtime = task.computing_options.get('container_runtime', None)

        if not container_runtime:

            container_runtime = task.computing.default_container_runtime

            container_engine = task.computing_options.get('container_engine', None)

        if not container_engine:

            container_engine = task.computing.default_container_engine



        # Runtime-specific part 

        if container_runtime == 'singularity':

        # engine-specific part 

        if container_engine == 'singularity':



            #if not task.container.supports_custom_interface_port:

            #     raise Exception('This task does not support dynamic port allocation and is therefore not supported using singularity on Slurm')
@@ -479,7 +479,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag 
            run_command+='docker://{}/{}:{} &> \$HOME/{}.log\\" > \$HOME/{}.sh && sbatch {} \$HOME/{}.sh"\''.format(task.container.registry, task.container.image_name, task.container.image_tag, task.uuid, task.uuid, sbatch_args, task.uuid)



        else:

            raise NotImplementedError('Container runtime {} not supported'.format(container_runtime))

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))



        out = os_shell(run_command, capture=True)

        if out.exit_code != 0:

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+3 −3
Original line number Diff line number Diff line
@@ -267,7 +267,7 @@ to provide help, news and informations on your deployment. Or you can just ignor 
                                     access_mode = 'internal',

                                     auth_mode = 'internal',

                                     wms = None,

                                     container_runtimes = ['docker'])

                                     container_engines = ['docker'])



            

            # Demo standalone computing plus conf
@@ -280,7 +280,7 @@ to provide help, news and informations on your deployment. Or you can just ignor 
                                                                 auth_mode = 'user_keys',

                                                                 wms = None,

                                                                 conf = {'host': 'standaloneworker'},

                                                                 container_runtimes = ['singularity','podman'])

                                                                 container_engines = ['singularity','podman'])

    

            # Add testuser extra conf for this computing resource

            testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_singlenode_computing, value= 'testuser')
@@ -295,7 +295,7 @@ to provide help, news and informations on your deployment. Or you can just ignor 
                                                            auth_mode = 'user_keys',

                                                            wms = 'slurm',

                                                            conf = {'host': 'slurmclustermaster', 'default_partition': 'partition1'},

                                                            container_runtimes = ['singularity'])

                                                            container_engines = ['singularity'])

           

            # Add testuser extra conf for this computing resource

            testuser.profile.add_extra_conf(conf_type = 'computing_user', object=demo_slurm_computing, value= 'slurmtestuser')

services/webapp/code/rosetta/core_app/migrations/0029_auto_20211218_2354.py

0 → 100644
+18 −0
Original line number Diff line number Diff line 
# Generated by Django 2.2.1 on 2021-12-18 23:54



from django.db import migrations





class Migration(migrations.Migration):



    dependencies = [

        ('core_app', '0028_computing_arch'),

    ]



    operations = [

        migrations.RenameField(

            model_name='computing',

            old_name='container_runtimes',

            new_name='container_engines',

        ),

    ]

services/webapp/code/rosetta/core_app/migrations/0030_auto_20211218_2355.py

0 → 100644
+19 −0
Original line number Diff line number Diff line 
# Generated by Django 2.2.1 on 2021-12-18 23:55



import django.contrib.postgres.fields.jsonb

from django.db import migrations





class Migration(migrations.Migration):



    dependencies = [

        ('core_app', '0029_auto_20211218_2354'),

    ]



    operations = [

        migrations.AlterField(

            model_name='computing',

            name='container_engines',

            field=django.contrib.postgres.fields.jsonb.JSONField(verbose_name='Container engines/runtimes'),

        ),

    ]