Commit d0b24086 authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Added Computing configurations and populate demo data. Added some testing and... 

Added Computing configurations and populate demo data. Added some testing and improving running tests with SQLite. Fixes.
parent 2cfb340a

run_webapp_unit_tests.sh

+5 −1
Original line number Diff line number Diff line
@@ -6,4 +6,8 @@ 
DJANGO_LOG_LEVEL="CRITICAL"

ROSETTA_LOG_LEVEL="CRITICAL"



rosetta/shell webapp "cd /opt/webapp_code && DJANGO_LOG_LEVEL=$DJANGO_LOG_LEVEL ROSETTA_LOG_LEVEL=$ROSETTA_LOG_LEVEL python3 manage.py test $@"
 
# Set DB to SQLIte in-memory

DJANGO_DB_ENGINE="django.db.backends.sqlite3"

DJANGO_DB_NAME=":memory:"



rosetta/shell webapp "export DJANGO_DB_ENGINE=$DJANGO_DB_ENGINE && export DJANGO_DB_NAME=$DJANGO_DB_NAME && cd /opt/webapp_code && python3 manage.py makemigrations && DJANGO_LOG_LEVEL=$DJANGO_LOG_LEVEL ROSETTA_LOG_LEVEL=$ROSETTA_LOG_LEVEL python3 manage.py test $@"

services/base/keys/id_rsa.pub

+1 −1
Original line number Diff line number Diff line 
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== docker@dev.ops
 
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== rosetta@rosetta.platform

services/webapp/code/rosetta/base_app/fields.py

0 → 100644
+29 −0
Original line number Diff line number Diff line 
import json

from django.db.models import Field



class JSONField(Field):

    def db_type(self, connection):

        return 'text'



    def from_db_value(self, value, expression, connection):

        if value is not None:

            return self.to_python(value)

        return value



    def to_python(self, value):

        if value is not None:

            try:

                return json.loads(value)

            except (TypeError, ValueError):

                return value

        return value



    def get_prep_value(self, value):

        if value is not None:

            return str(json.dumps(value))

        return value



    def value_to_string(self, obj):

        return self.value_from_object(obj)



# Credits: https://medium.com/@philamersune/using-postgresql-jsonfield-in-sqlite-95ad4ad2e5f1
 
 No newline at end of file

services/webapp/code/rosetta/base_app/management/commands/base_app_populate.py

+41 −13
Original line number Diff line number Diff line 
from django.core.management.base import BaseCommand

from django.contrib.auth.models import User

from ...models import Profile, Container, Computing

from ...models import Profile, Container, Computing, ComputingSysConf, ComputingUserConf



class Command(BaseCommand):

    help = 'Adds the admin superuser with \'a\' password.'
@@ -69,22 +69,50 @@ class Command(BaseCommand): 
                                     registry      = 'docker_hub',

                                     service_ports = '8888')







        # Computing resources

        #Computing.objects.create(user = None,

        #                         name = 'L',

        #                         type = '')



        # Computing resources

        #Computing.objects.create(user = None,

        #                         name = 'L',

        #                         type = '')

        computing_resources = Computing.objects.all()

        if computing_resources:

            print('Not creating demo computing resources as they already exist')

        else:

            print('Creating demo computing resources containers...')



            # Local computing resource

            Computing.objects.create(user = None,

                                     name = 'Local',

                                     type = 'local')

    

            # Demo remote computing resource

            demo_remote_computing = Computing.objects.create(user = None,

                                     name = 'Demo remote',

                                     type = 'remote',

                                     requires_sys_conf  = True,

                                     requires_user_conf = False)

    

            # Create demo remote sys computing conf

            ComputingSysConf.objects.create(computing = demo_remote_computing,

                                            data      = {'host': 'slurmclusterworker-one',

                                                         'user': 'rosetta',

                                                         'identity': 'privkey?'})





            # Demo slurm computing resource

            demo_slurm_computing = Computing.objects.create(user = None,

                                     name = 'Demo Slurm',

                                     type = 'slurm',

                                     requires_sys_conf  = True,

                                     requires_user_conf = True)

    

            # Create demo slurm sys computing conf

            ComputingSysConf.objects.create(computing = demo_slurm_computing,

                                            data      = {'master': 'slurmclusterworker-master'})



            # Create demo slurm user computing conf

            ComputingUserConf.objects.create(user      = testuser,

                                             computing = demo_slurm_computing,

                                             data      = {'user': 'testuser',

                                                          'id_rsa': '/rosetta/.ssh/id_rsa',

                                                          'id_rsa.pub': 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC2n4wiLiRmE1sla5+w0IW3wwPW/mqhhkm7IyCBS+rGTgnts7xsWcxobvamNdD6KSLNnjFZbBb7Yaf/BvWrwQgdqIFVU3gRWHYzoU6js+lKtBjd0e2DAVGivWCKEkSGLx7zhx7uH/Jt8kyZ4NaZq0p5+SFHBzePdR/1rURd8G8+G3OaCPKqP+JQT4RMUQHC5SNRJLcK1piYdmhDiYEyuQG4FlStKCWLCXeUY2EVirNMeQIfOgbUHJsVjH07zm1y8y7lTWDMWVZOnkG6Ap5kB+n4l1eWbslOKgDv29JTFOMU+bvGvYZh70lmLK7Hg4CMpXVgvw5VF9v97YiiigLwvC7wasBHaASwH7wUqakXYhdGFxJ23xVMSLnvJn4S++4L8t8bifRIVqhT6tZCPOU4fdOvJKCRjKrf7gcW/E33ovZFgoOCJ2vBLIh9N9ME0v7tG15JpRtgIBsCXwLcl3tVyCZJ/eyYMbc3QJGsbcPGb2CYRjDbevPCQlNavcMdlyrNIke7VimM5aW8OBJKVh5wCNRpd9XylrKo1cZHYxu/c5Lr6VUZjLpxDlSz+IuTn4VE7vmgHNPnXdlxRKjLHG/FZrZTSCWFEBcRoSa/hysLSFwwDjKd9nelOZRNBvJ+NY48vA8ixVnk4WAMlR/5qhjTRam66BVysHeRcbjJ2IGjwTJC5Q== rosetta@rosetta.platform'})



        # Computing resources

        #Computing.objects.create(user = None,

        #                         name = 'L',

        #                         type = '')

services/webapp/code/rosetta/base_app/models.py

+95 −3
Original line number Diff line number Diff line 
import uuid

import enum



from django.conf import settings

from django.db import models

from django.contrib.auth.models import User

from django.utils import timezone



from .utils import os_shell



if 'sqlite' in settings.DATABASES['default']['ENGINE']:

    from .fields import JSONField

else:

    from django.contrib.postgres.fields import JSONField



class ConfigurationError(Exception):

    pass



class ConsistencyError(Exception):

    pass





# Setup logging

import logging

logger = logging.getLogger(__name__)
@@ -90,6 +103,10 @@ class Computing(models.Model): 
    # If a compute resource has no user, it will be available to anyone. Can be created, edited and deleted only by admins.

    

    name = models.CharField('Computing Name', max_length=255, blank=False, null=False)

    type = models.CharField('Computing Type', max_length=255, blank=False, null=False)



    requires_sys_conf  = models.BooleanField(default=False)

    requires_user_conf = models.BooleanField(default=False)



    def __str__(self):

        return str('Computing Resource "{}" of user "{}"'.format(self.name, self.user))
@@ -98,6 +115,81 @@ class Computing(models.Model): 
    def id(self):

        return str(self.uuid).split('-')[0]



    # Validate conf

    def validate_conf_data(self, sys_conf_data=None, user_conf_data=None):

        

        if self.type == 'local':

            pass

        

        elif self.type == 'remote':

            # Check that we have all the data for a remote computing resource



            # Look for host:

            host_found = False

            if sys_conf_data  and 'host' in sys_conf_data  and sys_conf_data['host']:  host_found=True

            if user_conf_data and 'host' in user_conf_data and user_conf_data['host']: host_found=True

            if not host_found:

                raise ConfigurationError('Missing host in conf')

            

            

            # Look for user:

            user_found = False

            if sys_conf_data  and 'user' in sys_conf_data  and sys_conf_data['user']:  user_found=True

            if user_conf_data and 'user' in user_conf_data and user_conf_data['user']: user_found=True

            if not user_found:

                raise ConfigurationError('Missing user in conf')               



            # Look for password/identity:

            password_found = False

            identity_found = False

            if sys_conf_data  and 'password' in sys_conf_data  and sys_conf_data['password']:  password_found=True

            if user_conf_data and 'password' in user_conf_data and user_conf_data['password']: password_found=True

            if sys_conf_data  and 'identity' in sys_conf_data  and sys_conf_data['identity']:  identity_found=True

            if user_conf_data and 'identity' in user_conf_data and user_conf_data['identity']: identity_found=True       

            if not password_found and not identity_found:

                raise ConfigurationError('Missing password or identity in conf')



        elif self.type == 'slurm':

            raise NotImplementedError('Not yet implemented for Slurm')



        else:

            raise ConsistencyError('Unknown computing type "{}"'.format(self.type))

    

    @property    

    def sys_conf_data(self):          

        return ComputingSysConf.objects.get(computing=self).data

    

    #@property    

    #def user_conf_data(self):

    #    return {'testuser':'ciao'}

    

    def attach_user_conf_data(self, user):

        try:

            self.user_conf_data = ComputingUserConf.objects.get(computing=self).data

        except ComputingUserConf.DoesNotExist:

            self.user_conf_data = None





class ComputingSysConf(models.Model):

    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)

    computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)

    data = JSONField(blank=True, null=True)



    @property

    def id(self):

        return str(self.uuid).split('-')[0]





class ComputingUserConf(models.Model):

    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)

    user = models.ForeignKey(User, related_name='+', on_delete=models.CASCADE, null=True)

    computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)

    data = JSONField(blank=True, null=True)



    @property

    def id(self):

        return str(self.uuid).split('-')[0]





#=========================

#  Tasks
@@ -109,13 +201,13 @@ class Task(models.Model): 
    name      = models.CharField('Task name', max_length=36, blank=False, null=False)

    status    = models.CharField('Task status', max_length=36, blank=True, null=True)

    created   = models.DateTimeField('Created on', default=timezone.now)

    computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)

    pid       = models.IntegerField('Task pid', blank=True, null=True)

    port      = models.IntegerField('Task port', blank=True, null=True)

    ip        = models.CharField('Task ip address', max_length=36, blank=True, null=True)

    tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True)



    # Links

    computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)

    container = models.ForeignKey('Container', on_delete=models.CASCADE, related_name='+')



    def save(self, *args, **kwargs):